ensemble.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "350708b8-ac67-48db-ba1f-dd7d1ed195f2",
   "metadata": {},
   "source": [
    "## storagy check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c473ca36-fbf4-4050-8d33-13f94d49d8b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "!df -h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e02a461c-551a-43d1-a32a-61d3a24ac68a",
   "metadata": {},
   "outputs": [],
   "source": [
    "!du -h --max-depth=1 | sort -hr"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea181eb0-b3c9-4d92-a56e-8e35141e7b74",
   "metadata": {},
   "source": [
    "## Import library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "00971362-bb46-4132-aeef-a4c066599b63",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import gluonnlp as nlp\n",
    "from pathlib import Path\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from tqdm import tqdm, tqdm_notebook\n",
    "from kobert.utils import get_tokenizer\n",
    "from kobert.pytorch_kobert import get_pytorch_kobert_model\n",
    "from transformers import AdamW\n",
    "from transformers.optimization import get_cosine_schedule_with_warmup\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "import time\n",
    "import random\n",
    "import pickle\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "# plt.style.use('seaborn')\n",
    "\n",
    "import missingno as msno\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0636bbd4-b3cb-472e-844a-9f1c5ae6f530",
   "metadata": {},
   "source": [
    "## Fix seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "45f89ce7-6d7a-4995-9758-a1ec9ad439cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def seed_everything(seed):\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)  # if use multi-GPU\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "    np.random.seed(seed)\n",
    "    random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "caefea07-9069-4af9-97c2-6d40c51a59d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15b3e1be-4ecc-43a4-aff3-f86b5f1813f4",
   "metadata": {},
   "source": [
    "## Use cuda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "aa52809f-7e87-4740-9db7-6b4c895fac60",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda:0')\n",
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3060b4a-5b77-444a-9006-e8eb51b9ab71",
   "metadata": {},
   "source": [
    "## Hyper-parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "63bed85f-d7d4-41c2-8ce6-4f6e0d3702c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "max_len = 128\n",
    "batch_size = 32\n",
    "warmup_ratio = 0.01\n",
    "num_epochs = 30\n",
    "max_grad_norm = 1\n",
    "learning_rate =  5e-6\n",
    "stop_count = 5\n",
    "num_folds = 10\n",
    "PATH = './model/model_state_dict_init'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37314f2f-aca0-412f-a9cb-a3cd3727e623",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2480f37e-46f4-4619-b34c-ff6a4560b5a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "class RE_Dataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, tokenized_dataset, labels):\n",
    "        self.tokenized_dataset = tokenized_dataset\n",
    "        self.labels = labels\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.tokenized_dataset.items()}\n",
    "        item['labels'] = torch.tensor(self.labels[idx])\n",
    "        return item\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4804b715-80c2-4614-9e82-1a99c3b69c69",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocessing_dataset(dataset, label_type):\n",
    "    label = []\n",
    "    for i in dataset[8]:\n",
    "        if i == 'blind':\n",
    "            label.append(100)\n",
    "        else:\n",
    "            label.append(label_type[i])\n",
    "    out_dataset = pd.DataFrame({'0': dataset[0], 'sentence':dataset[1],'entity_01':dataset[2],'entity_02':dataset[5],'label':label,})\n",
    "    return out_dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f9647579-d62e-43a8-b3b3-449213b7e6b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(dataset_dir):\n",
    "  # load label_type, classes\n",
    "  with open('/opt/ml/input/data/label_type.pkl', 'rb') as f:\n",
    "    label_type = pickle.load(f)\n",
    "  # load dataset\n",
    "  dataset = pd.read_csv(dataset_dir, delimiter='\\t', header=None)\n",
    "  # preprecessing dataset\n",
    "  dataset = preprocessing_dataset(dataset, label_type)\n",
    "  \n",
    "  return dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "144ab237-6a2c-4069-bc1c-0c2750e16261",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenized_dataset(dataset, tokenizer):\n",
    "    concat_entity = []\n",
    "    for e01, e02 in zip(dataset['entity_01'], dataset['entity_02']):\n",
    "        temp = ''\n",
    "        temp = e01 + '</s>' + e02\n",
    "        concat_entity.append(temp)\n",
    "    tokenized_sentences = tokenizer(\n",
    "        concat_entity,\n",
    "        list(dataset['sentence']),\n",
    "        return_tensors=\"pt\",\n",
    "        padding=True,\n",
    "        truncation='only_second',\n",
    "        max_length=max_len,\n",
    "        add_special_tokens=True,\n",
    "    )\n",
    "    return tokenized_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2ef24ad5-5521-4a8e-ba05-1f148c6b664b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = load_data('/opt/ml/input/data/train/train.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9717db9e-f54a-496d-ba6e-c8223ce5f2fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8948"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 10개 미만 label 삭제\n",
    "df_train = df_train[~df_train['label'].isin((40, 37, 29, 41, 19, 18, 26, 28, 39))]\n",
    "len(df_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "343302ad-ff45-47c5-b318-8c87a09ed4ba",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7a2be289-a233-4f6c-b1e5-cb19818a97b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# accuracy 계산\n",
    "def calc_accuracy(X,Y):\n",
    "    max_vals, max_indices = torch.max(X, 1)\n",
    "    train_acc = (max_indices == Y).sum().data.cpu().numpy()/max_indices.size()[0]\n",
    "    return train_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "dbc5aaf4-7f49-4351-9bc3-3bcf3dc204c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "class LabelSmoothingLoss(nn.Module):\n",
    "    def __init__(self, classes=42, smoothing=0.0, dim=-1):\n",
    "        super(LabelSmoothingLoss, self).__init__()\n",
    "        self.confidence = 1.0 - smoothing\n",
    "        self.smoothing = smoothing\n",
    "        self.cls = classes\n",
    "        self.dim = dim\n",
    "\n",
    "    def forward(self, pred, target):\n",
    "        pred = pred.log_softmax(dim=self.dim)\n",
    "        with torch.no_grad():\n",
    "            true_dist = torch.zeros_like(pred)\n",
    "            true_dist.fill_(self.smoothing / (self.cls - 1))\n",
    "            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)\n",
    "        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0cd3cf7f-68c3-4b19-8ec2-a3f564c7e4fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import XLMRobertaTokenizer, XLMRobertaForSequenceClassification, XLMRobertaConfig\n",
    "\n",
    "MODEL_NAME = \"xlm-roberta-large\"\n",
    "tokenizer = XLMRobertaTokenizer.from_pretrained(MODEL_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1cf7e149-c11e-42df-82b6-1fa9306b86c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at xlm-roberta-large were not used when initializing XLMRobertaForSequenceClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
      "- This IS expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of XLMRobertaForSequenceClassification were not initialized from the model checkpoint at xlm-roberta-large and are newly initialized: ['classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.out_proj.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model_config = XLMRobertaConfig.from_pretrained(MODEL_NAME)\n",
    "model_config.num_labels = 42\n",
    "model = XLMRobertaForSequenceClassification.from_pretrained(MODEL_NAME, config=model_config).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "fcbbd594-5f62-41c1-81b4-babe1acce1bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 모델 저장 (fold마다 모델을 불러오기 위해)\n",
    "torch.save(model.state_dict(), PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "02df3253-edc5-4b0b-aad9-6e4bfa87e9db",
   "metadata": {},
   "outputs": [],
   "source": [
    "t_dataset = tokenized_dataset(df_train, tokenizer)\n",
    "t_label = df_train['label'].values\n",
    "dataset = RE_Dataset(t_dataset, t_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "5b04e4c4-5cd4-4dfb-9b84-5e4896ffecf7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FOLD 0\n",
      "==========\n",
      "Epoch 0/29\n",
      "----------\n",
      "train Loss: 2.3571 Acc: 0.4272\n",
      "val Loss: 1.7743 Acc: 0.4951\n",
      "epochs_val acc: 0.4951\n",
      "epochs_before_best acc: 0.0000\n",
      "epochs_after_best acc: 0.4951\n",
      "Training complete in 3.0m 48.7668092250824s\n",
      "\n",
      "Epoch 1/29\n",
      "----------\n",
      "train Loss: 1.5470 Acc: 0.5223\n",
      "val Loss: 0.9225 Acc: 0.6191\n",
      "epochs_val acc: 0.6191\n",
      "epochs_before_best acc: 0.4951\n",
      "epochs_after_best acc: 0.6191\n",
      "Training complete in 3.0m 48.88650941848755s\n",
      "\n",
      "Epoch 2/29\n",
      "----------\n",
      "train Loss: 1.0525 Acc: 0.6792\n",
      "val Loss: 0.6754 Acc: 0.7108\n",
      "epochs_val acc: 0.7108\n",
      "epochs_before_best acc: 0.6191\n",
      "epochs_after_best acc: 0.7108\n",
      "Training complete in 3.0m 48.48181748390198s\n",
      "\n",
      "Epoch 3/29\n",
      "----------\n",
      "train Loss: 0.8265 Acc: 0.7506\n",
      "val Loss: 0.6722 Acc: 0.7307\n",
      "epochs_val acc: 0.7307\n",
      "epochs_before_best acc: 0.7108\n",
      "epochs_after_best acc: 0.7307\n",
      "Training complete in 3.0m 48.6054413318634s\n",
      "\n",
      "Epoch 4/29\n",
      "----------\n",
      "train Loss: 0.9903 Acc: 0.7913\n",
      "val Loss: 1.3741 Acc: 0.7541\n",
      "epochs_val acc: 0.7541\n",
      "epochs_before_best acc: 0.7307\n",
      "epochs_after_best acc: 0.7541\n",
      "Training complete in 3.0m 48.1031277179718s\n",
      "\n",
      "Epoch 5/29\n",
      "----------\n",
      "train Loss: 0.6316 Acc: 0.8262\n",
      "val Loss: 0.8709 Acc: 0.7698\n",
      "epochs_val acc: 0.7698\n",
      "epochs_before_best acc: 0.7541\n",
      "epochs_after_best acc: 0.7698\n",
      "Training complete in 3.0m 48.74613094329834s\n",
      "\n",
      "Epoch 6/29\n",
      "----------\n",
      "train Loss: 0.2116 Acc: 0.8570\n",
      "val Loss: 0.4310 Acc: 0.7476\n",
      "Training complete in 3.0m 29.556875944137573s\n",
      "\n",
      "Epoch 7/29\n",
      "----------\n",
      "train Loss: 0.0854 Acc: 0.8822\n",
      "val Loss: 0.8176 Acc: 0.7485\n",
      "Training complete in 3.0m 29.383191347122192s\n",
      "\n",
      "Epoch 8/29\n",
      "----------\n",
      "train Loss: 0.9503 Acc: 0.8985\n",
      "val Loss: 1.4577 Acc: 0.7697\n",
      "Training complete in 3.0m 29.185723781585693s\n",
      "\n",
      "Epoch 9/29\n",
      "----------\n",
      "train Loss: 0.2634 Acc: 0.9150\n",
      "val Loss: 1.1207 Acc: 0.7619\n",
      "Training complete in 3.0m 28.935547828674316s\n",
      "\n",
      "Epoch 10/29\n",
      "----------\n",
      "train Loss: 0.0666 Acc: 0.9298\n",
      "val Loss: 0.5402 Acc: 0.7699\n",
      "epochs_val acc: 0.7699\n",
      "epochs_before_best acc: 0.7698\n",
      "epochs_after_best acc: 0.7699\n",
      "Training complete in 3.0m 47.80108022689819s\n",
      "\n",
      "Epoch 11/29\n",
      "----------\n",
      "train Loss: 0.0818 Acc: 0.9438\n",
      "val Loss: 0.5795 Acc: 0.7822\n",
      "epochs_val acc: 0.7822\n",
      "epochs_before_best acc: 0.7699\n",
      "epochs_after_best acc: 0.7822\n",
      "Training complete in 3.0m 47.74519205093384s\n",
      "\n",
      "Epoch 12/29\n",
      "----------\n",
      "train Loss: 0.2630 Acc: 0.9496\n",
      "val Loss: 0.5732 Acc: 0.7633\n",
      "Training complete in 3.0m 29.195322513580322s\n",
      "\n",
      "Epoch 13/29\n",
      "----------\n",
      "train Loss: 0.0858 Acc: 0.9590\n",
      "val Loss: 0.9249 Acc: 0.7789\n",
      "Training complete in 3.0m 28.31503176689148s\n",
      "\n",
      "Epoch 14/29\n",
      "----------\n",
      "train Loss: 0.0621 Acc: 0.9630\n",
      "val Loss: 1.0252 Acc: 0.7810\n",
      "Training complete in 3.0m 29.446858882904053s\n",
      "\n",
      "Epoch 15/29\n",
      "----------\n",
      "train Loss: 0.0989 Acc: 0.9688\n",
      "val Loss: 1.3060 Acc: 0.7731\n",
      "Training complete in 3.0m 30.81841731071472s\n",
      "\n",
      "Epoch 16/29\n",
      "----------\n",
      "train Loss: 0.1659 Acc: 0.9771\n",
      "val Loss: 0.7900 Acc: 0.7844\n",
      "epochs_val acc: 0.7844\n",
      "epochs_before_best acc: 0.7822\n",
      "epochs_after_best acc: 0.7844\n",
      "Training complete in 3.0m 48.65878915786743s\n",
      "\n",
      "Epoch 17/29\n",
      "----------\n",
      "train Loss: 0.0096 Acc: 0.9782\n",
      "val Loss: 0.6822 Acc: 0.7688\n",
      "Training complete in 3.0m 29.80225682258606s\n",
      "\n",
      "Epoch 18/29\n",
      "----------\n",
      "train Loss: 0.0061 Acc: 0.9800\n",
      "val Loss: 1.4351 Acc: 0.7598\n",
      "Training complete in 3.0m 29.28118634223938s\n",
      "\n",
      "Epoch 19/29\n",
      "----------\n",
      "train Loss: 0.0332 Acc: 0.9836\n",
      "val Loss: 1.2507 Acc: 0.7720\n",
      "Training complete in 3.0m 29.370750904083252s\n",
      "\n",
      "Epoch 20/29\n",
      "----------\n",
      "train Loss: 0.0734 Acc: 0.9839\n",
      "val Loss: 0.5973 Acc: 0.7700\n",
      "Training complete in 3.0m 28.943786144256592s\n",
      "\n",
      "Epoch 21/29\n",
      "----------\n",
      "train Loss: 0.0022 Acc: 0.9865\n",
      "val Loss: 1.1604 Acc: 0.7844\n",
      "Training complete in 3.0m 29.490153551101685s\n",
      "...Early Stopping...\n",
      "\n",
      "Best val Acc: 0.784418202764977\n",
      "FOLD 1\n",
      "==========\n",
      "Epoch 0/29\n",
      "----------\n",
      "train Loss: 1.7504 Acc: 0.4293\n",
      "val Loss: 2.5603 Acc: 0.4959\n",
      "epochs_val acc: 0.4959\n",
      "epochs_before_best acc: 0.0000\n",
      "epochs_after_best acc: 0.4959\n",
      "Training complete in 3.0m 48.06263327598572s\n",
      "\n",
      "Epoch 1/29\n",
      "----------\n",
      "train Loss: 1.9694 Acc: 0.5075\n",
      "val Loss: 1.3438 Acc: 0.5387\n",
      "epochs_val acc: 0.5387\n",
      "epochs_before_best acc: 0.4959\n",
      "epochs_after_best acc: 0.5387\n",
      "Training complete in 3.0m 48.14973497390747s\n",
      "\n",
      "Epoch 2/29\n",
      "----------\n",
      "train Loss: 1.0034 Acc: 0.6377\n",
      "val Loss: 1.3533 Acc: 0.7105\n",
      "epochs_val acc: 0.7105\n",
      "epochs_before_best acc: 0.5387\n",
      "epochs_after_best acc: 0.7105\n",
      "Training complete in 3.0m 47.899099349975586s\n",
      "\n",
      "Epoch 3/29\n",
      "----------\n",
      "train Loss: 0.7291 Acc: 0.7262\n",
      "val Loss: 1.2906 Acc: 0.7350\n",
      "epochs_val acc: 0.7350\n",
      "epochs_before_best acc: 0.7105\n",
      "epochs_after_best acc: 0.7350\n",
      "Training complete in 3.0m 47.83242464065552s\n",
      "\n",
      "Epoch 4/29\n",
      "----------\n",
      "train Loss: 1.2423 Acc: 0.7871\n",
      "val Loss: 1.0929 Acc: 0.7563\n",
      "epochs_val acc: 0.7563\n",
      "epochs_before_best acc: 0.7350\n",
      "epochs_after_best acc: 0.7563\n",
      "Training complete in 3.0m 48.997982025146484s\n",
      "\n",
      "Epoch 5/29\n",
      "----------\n",
      "train Loss: 0.5810 Acc: 0.8220\n",
      "val Loss: 0.9665 Acc: 0.7508\n",
      "Training complete in 3.0m 29.711733102798462s\n",
      "\n",
      "Epoch 6/29\n",
      "----------\n",
      "train Loss: 0.4928 Acc: 0.8522\n",
      "val Loss: 1.1706 Acc: 0.7507\n",
      "Training complete in 3.0m 29.58029580116272s\n",
      "\n",
      "Epoch 7/29\n",
      "----------\n",
      "train Loss: 0.9543 Acc: 0.8751\n",
      "val Loss: 1.2428 Acc: 0.7598\n",
      "epochs_val acc: 0.7598\n",
      "epochs_before_best acc: 0.7563\n",
      "epochs_after_best acc: 0.7598\n",
      "Training complete in 3.0m 48.231000661849976s\n",
      "\n",
      "Epoch 8/29\n",
      "----------\n",
      "train Loss: 0.4832 Acc: 0.8948\n",
      "val Loss: 0.5755 Acc: 0.7688\n",
      "epochs_val acc: 0.7688\n",
      "epochs_before_best acc: 0.7598\n",
      "epochs_after_best acc: 0.7688\n",
      "Training complete in 3.0m 48.6109037399292s\n",
      "\n",
      "Epoch 9/29\n",
      "----------\n",
      "train Loss: 0.1860 Acc: 0.9141\n",
      "val Loss: 1.0307 Acc: 0.7631\n",
      "Training complete in 3.0m 28.89182686805725s\n",
      "\n",
      "Epoch 10/29\n",
      "----------\n",
      "train Loss: 0.2088 Acc: 0.9267\n",
      "val Loss: 1.0441 Acc: 0.7743\n",
      "epochs_val acc: 0.7743\n",
      "epochs_before_best acc: 0.7688\n",
      "epochs_after_best acc: 0.7743\n",
      "Training complete in 3.0m 48.30094766616821s\n",
      "\n",
      "Epoch 11/29\n",
      "----------\n",
      "train Loss: 0.4822 Acc: 0.9417\n",
      "val Loss: 0.7082 Acc: 0.7777\n",
      "epochs_val acc: 0.7777\n",
      "epochs_before_best acc: 0.7743\n",
      "epochs_after_best acc: 0.7777\n",
      "Training complete in 3.0m 47.97431397438049s\n",
      "\n",
      "Epoch 12/29\n",
      "----------\n",
      "train Loss: 0.0196 Acc: 0.9519\n",
      "val Loss: 0.7859 Acc: 0.7788\n",
      "epochs_val acc: 0.7788\n",
      "epochs_before_best acc: 0.7777\n",
      "epochs_after_best acc: 0.7788\n",
      "Training complete in 3.0m 47.892152547836304s\n",
      "\n",
      "Epoch 13/29\n",
      "----------\n",
      "train Loss: 0.2721 Acc: 0.9586\n",
      "val Loss: 1.3487 Acc: 0.7753\n",
      "Training complete in 3.0m 28.753453969955444s\n",
      "\n",
      "Epoch 14/29\n",
      "----------\n",
      "train Loss: 0.1983 Acc: 0.9651\n",
      "val Loss: 1.6954 Acc: 0.7720\n",
      "Training complete in 3.0m 28.754940032958984s\n",
      "\n",
      "Epoch 15/29\n",
      "----------\n",
      "train Loss: 0.0619 Acc: 0.9690\n",
      "val Loss: 1.6208 Acc: 0.7854\n",
      "epochs_val acc: 0.7854\n",
      "epochs_before_best acc: 0.7788\n",
      "epochs_after_best acc: 0.7854\n",
      "Training complete in 3.0m 47.03310203552246s\n",
      "\n",
      "Epoch 16/29\n",
      "----------\n",
      "train Loss: 0.0657 Acc: 0.9764\n",
      "val Loss: 1.6873 Acc: 0.7776\n",
      "Training complete in 3.0m 28.876604557037354s\n",
      "\n",
      "Epoch 17/29\n",
      "----------\n",
      "train Loss: 0.1010 Acc: 0.9769\n",
      "val Loss: 0.5422 Acc: 0.7778\n",
      "Training complete in 3.0m 28.935725688934326s\n",
      "\n",
      "Epoch 18/29\n",
      "----------\n",
      "train Loss: 0.2133 Acc: 0.9801\n",
      "val Loss: 0.7339 Acc: 0.7722\n",
      "Training complete in 3.0m 28.170419216156006s\n",
      "\n",
      "Epoch 19/29\n",
      "----------\n",
      "train Loss: 0.0434 Acc: 0.9813\n",
      "val Loss: 0.4048 Acc: 0.7778\n",
      "Training complete in 3.0m 29.077608108520508s\n",
      "\n",
      "Epoch 20/29\n",
      "----------\n",
      "train Loss: 0.0256 Acc: 0.9838\n",
      "val Loss: 0.4527 Acc: 0.7801\n",
      "Training complete in 3.0m 28.944052934646606s\n",
      "...Early Stopping...\n",
      "\n",
      "Best val Acc: 0.785426267281106\n",
      "FOLD 2\n",
      "==========\n",
      "Epoch 0/29\n",
      "----------\n",
      "train Loss: 1.7337 Acc: 0.4269\n",
      "val Loss: 2.7901 Acc: 0.4959\n",
      "epochs_val acc: 0.4959\n",
      "epochs_before_best acc: 0.0000\n",
      "epochs_after_best acc: 0.4959\n",
      "Training complete in 3.0m 47.66333818435669s\n",
      "\n",
      "Epoch 1/29\n",
      "----------\n",
      "train Loss: 1.5820 Acc: 0.5247\n",
      "val Loss: 1.9135 Acc: 0.5988\n",
      "epochs_val acc: 0.5988\n",
      "epochs_before_best acc: 0.4959\n",
      "epochs_after_best acc: 0.5988\n",
      "Training complete in 3.0m 47.75723910331726s\n",
      "\n",
      "Epoch 2/29\n",
      "----------\n",
      "train Loss: 0.8369 Acc: 0.6627\n",
      "val Loss: 0.8501 Acc: 0.6894\n",
      "epochs_val acc: 0.6894\n",
      "epochs_before_best acc: 0.5988\n",
      "epochs_after_best acc: 0.6894\n",
      "Training complete in 3.0m 47.51957154273987s\n",
      "\n",
      "Epoch 3/29\n",
      "----------\n",
      "train Loss: 1.4518 Acc: 0.7386\n",
      "val Loss: 0.9107 Acc: 0.7519\n",
      "epochs_val acc: 0.7519\n",
      "epochs_before_best acc: 0.6894\n",
      "epochs_after_best acc: 0.7519\n",
      "Training complete in 3.0m 47.56366467475891s\n",
      "\n",
      "Epoch 4/29\n",
      "----------\n",
      "train Loss: 0.7697 Acc: 0.7872\n",
      "val Loss: 0.7919 Acc: 0.7509\n",
      "Training complete in 3.0m 29.02448058128357s\n",
      "\n",
      "Epoch 5/29\n",
      "----------\n",
      "train Loss: 0.4646 Acc: 0.8217\n",
      "val Loss: 0.5835 Acc: 0.7610\n",
      "epochs_val acc: 0.7610\n",
      "epochs_before_best acc: 0.7519\n",
      "epochs_after_best acc: 0.7610\n",
      "Training complete in 3.0m 47.36909747123718s\n",
      "\n",
      "Epoch 6/29\n",
      "----------\n",
      "train Loss: 0.4683 Acc: 0.8566\n",
      "val Loss: 0.7229 Acc: 0.7766\n",
      "epochs_val acc: 0.7766\n",
      "epochs_before_best acc: 0.7610\n",
      "epochs_after_best acc: 0.7766\n",
      "Training complete in 3.0m 47.96237850189209s\n",
      "\n",
      "Epoch 7/29\n",
      "----------\n",
      "train Loss: 0.3240 Acc: 0.8741\n",
      "val Loss: 0.5694 Acc: 0.7843\n",
      "epochs_val acc: 0.7843\n",
      "epochs_before_best acc: 0.7766\n",
      "epochs_after_best acc: 0.7843\n",
      "Training complete in 3.0m 48.140138387680054s\n",
      "\n",
      "Epoch 8/29\n",
      "----------\n",
      "train Loss: 0.2937 Acc: 0.8966\n",
      "val Loss: 1.1301 Acc: 0.7743\n",
      "Training complete in 3.0m 29.53534722328186s\n",
      "\n",
      "Epoch 9/29\n",
      "----------\n",
      "train Loss: 0.3834 Acc: 0.9113\n",
      "val Loss: 0.6584 Acc: 0.7867\n",
      "epochs_val acc: 0.7867\n",
      "epochs_before_best acc: 0.7843\n",
      "epochs_after_best acc: 0.7867\n",
      "Training complete in 3.0m 47.82519745826721s\n",
      "\n",
      "Epoch 10/29\n",
      "----------\n",
      "train Loss: 0.7105 Acc: 0.9303\n",
      "val Loss: 0.8196 Acc: 0.7889\n",
      "epochs_val acc: 0.7889\n",
      "epochs_before_best acc: 0.7867\n",
      "epochs_after_best acc: 0.7889\n",
      "Training complete in 3.0m 47.7176513671875s\n",
      "\n",
      "Epoch 11/29\n",
      "----------\n",
      "train Loss: 0.1035 Acc: 0.9426\n",
      "val Loss: 0.8101 Acc: 0.7766\n",
      "Training complete in 3.0m 29.43786096572876s\n",
      "\n",
      "Epoch 12/29\n",
      "----------\n",
      "train Loss: 0.5566 Acc: 0.9487\n",
      "val Loss: 1.6261 Acc: 0.7944\n",
      "epochs_val acc: 0.7944\n",
      "epochs_before_best acc: 0.7889\n",
      "epochs_after_best acc: 0.7944\n",
      "Training complete in 3.0m 47.736223459243774s\n",
      "\n",
      "Epoch 13/29\n",
      "----------\n",
      "train Loss: 0.3817 Acc: 0.9588\n",
      "val Loss: 0.8562 Acc: 0.7754\n",
      "Training complete in 3.0m 28.850865125656128s\n",
      "\n",
      "Epoch 14/29\n",
      "----------\n",
      "train Loss: 0.0546 Acc: 0.9622\n",
      "val Loss: 1.2427 Acc: 0.7754\n",
      "Training complete in 3.0m 28.900180101394653s\n",
      "\n",
      "Epoch 15/29\n",
      "----------\n",
      "train Loss: 0.2150 Acc: 0.9667\n",
      "val Loss: 0.7205 Acc: 0.7967\n",
      "epochs_val acc: 0.7967\n",
      "epochs_before_best acc: 0.7944\n",
      "epochs_after_best acc: 0.7967\n",
      "Training complete in 3.0m 47.41980719566345s\n",
      "\n",
      "Epoch 16/29\n",
      "----------\n",
      "train Loss: 0.2956 Acc: 0.9717\n",
      "val Loss: 0.8642 Acc: 0.7878\n",
      "Training complete in 3.0m 28.74296283721924s\n",
      "\n",
      "Epoch 17/29\n",
      "----------\n",
      "train Loss: 0.0111 Acc: 0.9757\n",
      "val Loss: 1.0349 Acc: 0.7866\n",
      "Training complete in 3.0m 28.616761207580566s\n",
      "\n",
      "Epoch 18/29\n",
      "----------\n",
      "train Loss: 0.3459 Acc: 0.9802\n",
      "val Loss: 0.7284 Acc: 0.7934\n",
      "Training complete in 3.0m 28.832257747650146s\n",
      "\n",
      "Epoch 19/29\n",
      "----------\n",
      "train Loss: 0.0053 Acc: 0.9798\n",
      "val Loss: 2.5467 Acc: 0.7964\n",
      "Training complete in 3.0m 28.761601209640503s\n",
      "\n",
      "Epoch 20/29\n",
      "----------\n",
      "train Loss: 0.2127 Acc: 0.9833\n",
      "val Loss: 1.0856 Acc: 0.7888\n",
      "Training complete in 3.0m 28.47899866104126s\n",
      "...Early Stopping...\n",
      "\n",
      "Best val Acc: 0.7966589861751151\n",
      "FOLD 3\n",
      "==========\n",
      "Epoch 0/29\n",
      "----------\n",
      "train Loss: 1.9634 Acc: 0.4267\n",
      "val Loss: 2.6508 Acc: 0.4948\n",
      "epochs_val acc: 0.4948\n",
      "epochs_before_best acc: 0.0000\n",
      "epochs_after_best acc: 0.4948\n",
      "Training complete in 3.0m 47.776599645614624s\n",
      "\n",
      "Epoch 1/29\n",
      "----------\n",
      "train Loss: 1.5016 Acc: 0.5020\n",
      "val Loss: 1.9376 Acc: 0.5553\n",
      "epochs_val acc: 0.5553\n",
      "epochs_before_best acc: 0.4948\n",
      "epochs_after_best acc: 0.5553\n",
      "Training complete in 3.0m 47.76375770568848s\n",
      "\n",
      "Epoch 2/29\n",
      "----------\n",
      "train Loss: 0.8871 Acc: 0.6112\n",
      "val Loss: 0.9759 Acc: 0.6716\n",
      "epochs_val acc: 0.6716\n",
      "epochs_before_best acc: 0.5553\n",
      "epochs_after_best acc: 0.6716\n",
      "Training complete in 3.0m 47.46342897415161s\n",
      "\n",
      "Epoch 3/29\n",
      "----------\n",
      "train Loss: 1.1845 Acc: 0.7062\n",
      "val Loss: 1.0648 Acc: 0.7117\n",
      "epochs_val acc: 0.7117\n",
      "epochs_before_best acc: 0.6716\n",
      "epochs_after_best acc: 0.7117\n",
      "Training complete in 3.0m 47.474209785461426s\n",
      "\n",
      "Epoch 4/29\n",
      "----------\n",
      "train Loss: 0.6124 Acc: 0.7605\n",
      "val Loss: 0.7523 Acc: 0.7039\n",
      "Training complete in 3.0m 29.383238077163696s\n",
      "\n",
      "Epoch 5/29\n",
      "----------\n",
      "train Loss: 0.1937 Acc: 0.8067\n",
      "val Loss: 0.6934 Acc: 0.7330\n",
      "epochs_val acc: 0.7330\n",
      "epochs_before_best acc: 0.7117\n",
      "epochs_after_best acc: 0.7330\n",
      "Training complete in 3.0m 47.861658811569214s\n",
      "\n",
      "Epoch 6/29\n",
      "----------\n",
      "train Loss: 0.9961 Acc: 0.8299\n",
      "val Loss: 1.0577 Acc: 0.7553\n",
      "epochs_val acc: 0.7553\n",
      "epochs_before_best acc: 0.7330\n",
      "epochs_after_best acc: 0.7553\n",
      "Training complete in 3.0m 47.525949239730835s\n",
      "\n",
      "Epoch 7/29\n",
      "----------\n",
      "train Loss: 0.0879 Acc: 0.8528\n",
      "val Loss: 0.9473 Acc: 0.7441\n",
      "Training complete in 3.0m 28.90217089653015s\n",
      "\n",
      "Epoch 8/29\n",
      "----------\n",
      "train Loss: 0.5472 Acc: 0.8742\n",
      "val Loss: 0.6495 Acc: 0.7476\n",
      "Training complete in 3.0m 29.106623888015747s\n",
      "\n",
      "Epoch 9/29\n",
      "----------\n",
      "train Loss: 0.4925 Acc: 0.8942\n",
      "val Loss: 0.2875 Acc: 0.7633\n",
      "epochs_val acc: 0.7633\n",
      "epochs_before_best acc: 0.7553\n",
      "epochs_after_best acc: 0.7633\n",
      "Training complete in 3.0m 47.823737144470215s\n",
      "\n",
      "Epoch 10/29\n",
      "----------\n",
      "train Loss: 0.1795 Acc: 0.9109\n",
      "val Loss: 0.8386 Acc: 0.7520\n",
      "Training complete in 3.0m 29.07957673072815s\n",
      "\n",
      "Epoch 11/29\n",
      "----------\n",
      "train Loss: 0.0405 Acc: 0.9203\n",
      "val Loss: 0.7790 Acc: 0.7554\n",
      "Training complete in 3.0m 29.075260877609253s\n",
      "\n",
      "Epoch 12/29\n",
      "----------\n",
      "train Loss: 0.2584 Acc: 0.9286\n",
      "val Loss: 0.4770 Acc: 0.7533\n",
      "Training complete in 3.0m 28.546528577804565s\n",
      "\n",
      "Epoch 13/29\n",
      "----------\n",
      "train Loss: 0.2205 Acc: 0.9431\n",
      "val Loss: 0.8276 Acc: 0.7576\n",
      "Training complete in 3.0m 29.543494939804077s\n",
      "\n",
      "Epoch 14/29\n",
      "----------\n",
      "train Loss: 0.0781 Acc: 0.9504\n",
      "val Loss: 1.6311 Acc: 0.7529\n",
      "Training complete in 3.0m 28.71010136604309s\n",
      "...Early Stopping...\n",
      "\n",
      "Best val Acc: 0.7632848502304147\n",
      "FOLD 4\n",
      "==========\n",
      "Epoch 0/29\n",
      "----------\n",
      "train Loss: 2.0368 Acc: 0.4286\n",
      "val Loss: 1.8170 Acc: 0.4950\n",
      "epochs_val acc: 0.4950\n",
      "epochs_before_best acc: 0.0000\n",
      "epochs_after_best acc: 0.4950\n",
      "Training complete in 3.0m 47.110440731048584s\n",
      "\n",
      "Epoch 1/29\n",
      "----------\n",
      "train Loss: 0.9902 Acc: 0.5149\n",
      "val Loss: 1.3236 Acc: 0.5711\n",
      "epochs_val acc: 0.5711\n",
      "epochs_before_best acc: 0.4950\n",
      "epochs_after_best acc: 0.5711\n",
      "Training complete in 3.0m 47.45969223976135s\n",
      "\n",
      "Epoch 2/29\n",
      "----------\n",
      "train Loss: 1.1689 Acc: 0.6415\n",
      "val Loss: 0.9080 Acc: 0.7028\n",
      "epochs_val acc: 0.7028\n",
      "epochs_before_best acc: 0.5711\n",
      "epochs_after_best acc: 0.7028\n",
      "Training complete in 3.0m 47.580782413482666s\n",
      "\n",
      "Epoch 3/29\n",
      "----------\n",
      "train Loss: 1.3753 Acc: 0.7348\n",
      "val Loss: 0.9355 Acc: 0.7207\n",
      "epochs_val acc: 0.7207\n",
      "epochs_before_best acc: 0.7028\n",
      "epochs_after_best acc: 0.7207\n",
      "Training complete in 3.0m 47.79377269744873s\n",
      "\n",
      "Epoch 4/29\n",
      "----------\n",
      "train Loss: 1.1645 Acc: 0.7833\n",
      "val Loss: 0.5662 Acc: 0.7497\n",
      "epochs_val acc: 0.7497\n",
      "epochs_before_best acc: 0.7207\n",
      "epochs_after_best acc: 0.7497\n",
      "Training complete in 3.0m 47.37838363647461s\n",
      "\n",
      "Epoch 5/29\n",
      "----------\n",
      "train Loss: 0.6012 Acc: 0.8194\n",
      "val Loss: 0.3354 Acc: 0.7309\n",
      "Training complete in 3.0m 28.759660959243774s\n",
      "\n",
      "Epoch 6/29\n",
      "----------\n",
      "train Loss: 0.4007 Acc: 0.8450\n",
      "val Loss: 0.7893 Acc: 0.7598\n",
      "epochs_val acc: 0.7598\n",
      "epochs_before_best acc: 0.7497\n",
      "epochs_after_best acc: 0.7598\n",
      "Training complete in 3.0m 47.613139390945435s\n",
      "\n",
      "Epoch 7/29\n",
      "----------\n",
      "train Loss: 0.6929 Acc: 0.8744\n",
      "val Loss: 0.6342 Acc: 0.7476\n",
      "Training complete in 3.0m 29.34067177772522s\n",
      "\n",
      "Epoch 8/29\n",
      "----------\n",
      "train Loss: 0.1936 Acc: 0.8940\n",
      "val Loss: 0.8225 Acc: 0.7621\n",
      "epochs_val acc: 0.7621\n",
      "epochs_before_best acc: 0.7598\n",
      "epochs_after_best acc: 0.7621\n",
      "Training complete in 3.0m 47.30450367927551s\n",
      "\n",
      "Epoch 9/29\n",
      "----------\n",
      "train Loss: 0.5914 Acc: 0.9116\n",
      "val Loss: 0.8912 Acc: 0.7575\n",
      "Training complete in 3.0m 28.613693475723267s\n",
      "\n",
      "Epoch 10/29\n",
      "----------\n",
      "train Loss: 0.0994 Acc: 0.9247\n",
      "val Loss: 1.3026 Acc: 0.7608\n",
      "Training complete in 3.0m 29.144569873809814s\n",
      "\n",
      "Epoch 11/29\n",
      "----------\n",
      "train Loss: 0.0877 Acc: 0.9381\n",
      "val Loss: 1.3225 Acc: 0.7564\n",
      "Training complete in 3.0m 28.40385150909424s\n",
      "\n",
      "Epoch 12/29\n",
      "----------\n",
      "train Loss: 0.0275 Acc: 0.9493\n",
      "val Loss: 1.8162 Acc: 0.7631\n",
      "epochs_val acc: 0.7631\n",
      "epochs_before_best acc: 0.7621\n",
      "epochs_after_best acc: 0.7631\n",
      "Training complete in 3.0m 47.556347370147705s\n",
      "\n",
      "Epoch 13/29\n",
      "----------\n",
      "train Loss: 0.2151 Acc: 0.9581\n",
      "val Loss: 1.2155 Acc: 0.7720\n",
      "epochs_val acc: 0.7720\n",
      "epochs_before_best acc: 0.7631\n",
      "epochs_after_best acc: 0.7720\n",
      "Training complete in 3.0m 47.73432755470276s\n",
      "\n",
      "Epoch 14/29\n",
      "----------\n",
      "train Loss: 0.0483 Acc: 0.9658\n",
      "val Loss: 1.2021 Acc: 0.7531\n",
      "Training complete in 3.0m 29.123955726623535s\n",
      "\n",
      "Epoch 15/29\n",
      "----------\n",
      "train Loss: 0.1156 Acc: 0.9696\n",
      "val Loss: 1.5925 Acc: 0.7731\n",
      "epochs_val acc: 0.7731\n",
      "epochs_before_best acc: 0.7720\n",
      "epochs_after_best acc: 0.7731\n",
      "Training complete in 3.0m 47.24635648727417s\n",
      "\n",
      "Epoch 16/29\n",
      "----------\n",
      "train Loss: 0.0748 Acc: 0.9714\n",
      "val Loss: 1.4234 Acc: 0.7654\n",
      "Training complete in 3.0m 29.318557739257812s\n",
      "\n",
      "Epoch 17/29\n",
      "----------\n",
      "train Loss: 0.2341 Acc: 0.9772\n",
      "val Loss: 1.9391 Acc: 0.7653\n",
      "Training complete in 3.0m 29.175716400146484s\n",
      "\n",
      "Epoch 18/29\n",
      "----------\n",
      "train Loss: 0.2092 Acc: 0.9784\n",
      "val Loss: 2.1925 Acc: 0.7496\n",
      "Training complete in 3.0m 28.992531538009644s\n",
      "\n",
      "Epoch 19/29\n",
      "----------\n",
      "train Loss: 0.0342 Acc: 0.9828\n",
      "val Loss: 1.4148 Acc: 0.7620\n",
      "Training complete in 3.0m 28.96527123451233s\n",
      "\n",
      "Epoch 20/29\n",
      "----------\n",
      "train Loss: 0.0780 Acc: 0.9852\n",
      "val Loss: 2.6786 Acc: 0.7652\n",
      "Training complete in 3.0m 29.016538619995117s\n",
      "...Early Stopping...\n",
      "\n",
      "Best val Acc: 0.7731494815668203\n",
      "FOLD 5\n",
      "==========\n",
      "Epoch 0/29\n",
      "----------\n",
      "train Loss: 2.4050 Acc: 0.4305\n",
      "val Loss: 2.5836 Acc: 0.4949\n",
      "epochs_val acc: 0.4949\n",
      "epochs_before_best acc: 0.0000\n",
      "epochs_after_best acc: 0.4949\n",
      "Training complete in 3.0m 47.89094662666321s\n",
      "\n",
      "Epoch 1/29\n",
      "----------\n",
      "train Loss: 2.0798 Acc: 0.4978\n",
      "val Loss: 1.3513 Acc: 0.5331\n",
      "epochs_val acc: 0.5331\n",
      "epochs_before_best acc: 0.4949\n",
      "epochs_after_best acc: 0.5331\n",
      "Training complete in 3.0m 47.69896483421326s\n",
      "\n",
      "Epoch 2/29\n",
      "----------\n",
      "train Loss: 1.4414 Acc: 0.6046\n",
      "val Loss: 1.4246 Acc: 0.6771\n",
      "epochs_val acc: 0.6771\n",
      "epochs_before_best acc: 0.5331\n",
      "epochs_after_best acc: 0.6771\n",
      "Training complete in 3.0m 47.73763585090637s\n",
      "\n",
      "Epoch 3/29\n",
      "----------\n",
      "train Loss: 1.0685 Acc: 0.7031\n",
      "val Loss: 1.0041 Acc: 0.7173\n",
      "epochs_val acc: 0.7173\n",
      "epochs_before_best acc: 0.6771\n",
      "epochs_after_best acc: 0.7173\n",
      "Training complete in 3.0m 47.45585083961487s\n",
      "\n",
      "Epoch 4/29\n",
      "----------\n",
      "train Loss: 0.6734 Acc: 0.7670\n",
      "val Loss: 0.5660 Acc: 0.7353\n",
      "epochs_val acc: 0.7353\n",
      "epochs_before_best acc: 0.7173\n",
      "epochs_after_best acc: 0.7353\n",
      "Training complete in 3.0m 48.04584503173828s\n",
      "\n",
      "Epoch 5/29\n",
      "----------\n",
      "train Loss: 1.1751 Acc: 0.7995\n",
      "val Loss: 0.4704 Acc: 0.7666\n",
      "epochs_val acc: 0.7666\n",
      "epochs_before_best acc: 0.7353\n",
      "epochs_after_best acc: 0.7666\n",
      "Training complete in 3.0m 47.49275064468384s\n",
      "\n",
      "Epoch 6/29\n",
      "----------\n",
      "train Loss: 0.7867 Acc: 0.8317\n",
      "val Loss: 0.7570 Acc: 0.7307\n",
      "Training complete in 3.0m 29.176055908203125s\n",
      "\n",
      "Epoch 7/29\n",
      "----------\n",
      "train Loss: 0.3735 Acc: 0.8564\n",
      "val Loss: 0.9256 Acc: 0.7441\n",
      "Training complete in 3.0m 28.78653597831726s\n",
      "\n",
      "Epoch 8/29\n",
      "----------\n",
      "train Loss: 0.4402 Acc: 0.8851\n",
      "val Loss: 1.0329 Acc: 0.7586\n",
      "Training complete in 3.0m 28.896585702896118s\n",
      "\n",
      "Epoch 9/29\n",
      "----------\n",
      "train Loss: 0.0735 Acc: 0.8987\n",
      "val Loss: 0.2566 Acc: 0.7801\n",
      "epochs_val acc: 0.7801\n",
      "epochs_before_best acc: 0.7666\n",
      "epochs_after_best acc: 0.7801\n",
      "Training complete in 3.0m 47.68125557899475s\n",
      "\n",
      "Epoch 10/29\n",
      "----------\n",
      "train Loss: 0.4382 Acc: 0.9173\n",
      "val Loss: 1.4834 Acc: 0.7653\n",
      "Training complete in 3.0m 28.60064435005188s\n",
      "\n",
      "Epoch 11/29\n",
      "----------\n",
      "train Loss: 0.0430 Acc: 0.9284\n",
      "val Loss: 0.9918 Acc: 0.7631\n",
      "Training complete in 3.0m 28.974456787109375s\n",
      "\n",
      "Epoch 12/29\n",
      "----------\n",
      "train Loss: 0.2803 Acc: 0.9434\n",
      "val Loss: 0.6950 Acc: 0.7711\n",
      "Training complete in 3.0m 28.852844715118408s\n",
      "\n",
      "Epoch 13/29\n",
      "----------\n",
      "train Loss: 0.4112 Acc: 0.9516\n",
      "val Loss: 1.2677 Acc: 0.7933\n",
      "epochs_val acc: 0.7933\n",
      "epochs_before_best acc: 0.7801\n",
      "epochs_after_best acc: 0.7933\n",
      "Training complete in 3.0m 48.39417219161987s\n",
      "\n",
      "Epoch 14/29\n",
      "----------\n",
      "train Loss: 0.0286 Acc: 0.9627\n",
      "val Loss: 0.9811 Acc: 0.7721\n",
      "Training complete in 3.0m 29.05778217315674s\n",
      "\n",
      "Epoch 15/29\n",
      "----------\n",
      "train Loss: 0.0121 Acc: 0.9633\n",
      "val Loss: 0.6960 Acc: 0.7599\n",
      "Training complete in 3.0m 29.202641487121582s\n",
      "\n",
      "Epoch 16/29\n",
      "----------\n",
      "train Loss: 0.1777 Acc: 0.9664\n",
      "val Loss: 1.9953 Acc: 0.7708\n",
      "Training complete in 3.0m 29.19229292869568s\n",
      "\n",
      "Epoch 17/29\n",
      "----------\n",
      "train Loss: 0.0115 Acc: 0.9702\n",
      "val Loss: 0.6939 Acc: 0.7699\n",
      "Training complete in 3.0m 29.04521679878235s\n",
      "\n",
      "Epoch 18/29\n",
      "----------\n",
      "train Loss: 0.1036 Acc: 0.9758\n",
      "val Loss: 1.7982 Acc: 0.7787\n",
      "Training complete in 3.0m 28.82598114013672s\n",
      "...Early Stopping...\n",
      "\n",
      "Best val Acc: 0.7933107718894009\n",
      "FOLD 6\n",
      "==========\n",
      "Epoch 0/29\n",
      "----------\n",
      "train Loss: 2.3713 Acc: 0.4285\n",
      "val Loss: 2.1278 Acc: 0.4950\n",
      "epochs_val acc: 0.4950\n",
      "epochs_before_best acc: 0.0000\n",
      "epochs_after_best acc: 0.4950\n",
      "Training complete in 3.0m 47.360804319381714s\n",
      "\n",
      "Epoch 1/29\n",
      "----------\n",
      "train Loss: 2.2761 Acc: 0.5178\n",
      "val Loss: 1.0853 Acc: 0.5947\n",
      "epochs_val acc: 0.5947\n",
      "epochs_before_best acc: 0.4950\n",
      "epochs_after_best acc: 0.5947\n",
      "Training complete in 3.0m 48.146790504455566s\n",
      "\n",
      "Epoch 2/29\n",
      "----------\n",
      "train Loss: 1.2913 Acc: 0.6531\n",
      "val Loss: 0.9360 Acc: 0.7027\n",
      "epochs_val acc: 0.7027\n",
      "epochs_before_best acc: 0.5947\n",
      "epochs_after_best acc: 0.7027\n",
      "Training complete in 3.0m 47.290064334869385s\n",
      "\n",
      "Epoch 3/29\n",
      "----------\n",
      "train Loss: 0.9094 Acc: 0.7290\n",
      "val Loss: 0.8451 Acc: 0.7274\n",
      "epochs_val acc: 0.7274\n",
      "epochs_before_best acc: 0.7027\n",
      "epochs_after_best acc: 0.7274\n",
      "Training complete in 3.0m 47.89385938644409s\n",
      "\n",
      "Epoch 4/29\n",
      "----------\n",
      "train Loss: 1.3301 Acc: 0.7831\n",
      "val Loss: 0.7369 Acc: 0.7587\n",
      "epochs_val acc: 0.7587\n",
      "epochs_before_best acc: 0.7274\n",
      "epochs_after_best acc: 0.7587\n",
      "Training complete in 3.0m 47.736459255218506s\n",
      "\n",
      "Epoch 5/29\n",
      "----------\n",
      "train Loss: 0.6109 Acc: 0.8166\n",
      "val Loss: 0.7936 Acc: 0.7520\n",
      "Training complete in 3.0m 28.647871732711792s\n",
      "\n",
      "Epoch 6/29\n",
      "----------\n",
      "train Loss: 0.6712 Acc: 0.8484\n",
      "val Loss: 0.6683 Acc: 0.7531\n",
      "Training complete in 3.0m 28.624762058258057s\n",
      "\n",
      "Epoch 7/29\n",
      "----------\n",
      "train Loss: 0.4948 Acc: 0.8658\n",
      "val Loss: 1.0845 Acc: 0.7641\n",
      "epochs_val acc: 0.7641\n",
      "epochs_before_best acc: 0.7587\n",
      "epochs_after_best acc: 0.7641\n",
      "Training complete in 3.0m 47.6567599773407s\n",
      "\n",
      "Epoch 8/29\n",
      "----------\n",
      "train Loss: 0.5438 Acc: 0.8855\n",
      "val Loss: 1.3408 Acc: 0.7742\n",
      "epochs_val acc: 0.7742\n",
      "epochs_before_best acc: 0.7641\n",
      "epochs_after_best acc: 0.7742\n",
      "Training complete in 3.0m 47.95018267631531s\n",
      "\n",
      "Epoch 9/29\n",
      "----------\n",
      "train Loss: 0.4758 Acc: 0.9046\n",
      "val Loss: 0.5913 Acc: 0.7744\n",
      "epochs_val acc: 0.7744\n",
      "epochs_before_best acc: 0.7742\n",
      "epochs_after_best acc: 0.7744\n",
      "Training complete in 3.0m 47.58727025985718s\n",
      "\n",
      "Epoch 10/29\n",
      "----------\n",
      "train Loss: 0.1153 Acc: 0.9183\n",
      "val Loss: 2.1014 Acc: 0.7663\n",
      "Training complete in 3.0m 28.840540885925293s\n",
      "\n",
      "Epoch 11/29\n",
      "----------\n",
      "train Loss: 0.1867 Acc: 0.9336\n",
      "val Loss: 0.8833 Acc: 0.7688\n",
      "Training complete in 3.0m 28.675881147384644s\n",
      "\n",
      "Epoch 12/29\n",
      "----------\n",
      "train Loss: 0.3723 Acc: 0.9427\n",
      "val Loss: 0.7590 Acc: 0.7687\n",
      "Training complete in 3.0m 29.37216854095459s\n",
      "\n",
      "Epoch 13/29\n",
      "----------\n",
      "train Loss: 0.1584 Acc: 0.9495\n",
      "val Loss: 1.0697 Acc: 0.7564\n",
      "Training complete in 3.0m 28.86082935333252s\n",
      "\n",
      "Epoch 14/29\n",
      "----------\n",
      "train Loss: 0.1854 Acc: 0.9580\n",
      "val Loss: 0.6621 Acc: 0.7655\n",
      "Training complete in 3.0m 29.102519273757935s\n",
      "...Early Stopping...\n",
      "\n",
      "Best val Acc: 0.7743735599078342\n",
      "FOLD 7\n",
      "==========\n",
      "Epoch 0/29\n",
      "----------\n",
      "train Loss: 2.3455 Acc: 0.4280\n",
      "val Loss: 1.9919 Acc: 0.4951\n",
      "epochs_val acc: 0.4951\n",
      "epochs_before_best acc: 0.0000\n",
      "epochs_after_best acc: 0.4951\n",
      "Training complete in 3.0m 47.63252878189087s\n",
      "\n",
      "Epoch 1/29\n",
      "----------\n",
      "train Loss: 1.6984 Acc: 0.5013\n",
      "val Loss: 2.4569 Acc: 0.5428\n",
      "epochs_val acc: 0.5428\n",
      "epochs_before_best acc: 0.4951\n",
      "epochs_after_best acc: 0.5428\n",
      "Training complete in 3.0m 47.97425699234009s\n",
      "\n",
      "Epoch 2/29\n",
      "----------\n",
      "train Loss: 0.8919 Acc: 0.6261\n",
      "val Loss: 1.3880 Acc: 0.6804\n",
      "epochs_val acc: 0.6804\n",
      "epochs_before_best acc: 0.5428\n",
      "epochs_after_best acc: 0.6804\n",
      "Training complete in 3.0m 47.98112964630127s\n",
      "\n",
      "Epoch 3/29\n",
      "----------\n",
      "train Loss: 1.2312 Acc: 0.7177\n",
      "val Loss: 0.8148 Acc: 0.7242\n",
      "epochs_val acc: 0.7242\n",
      "epochs_before_best acc: 0.6804\n",
      "epochs_after_best acc: 0.7242\n",
      "Training complete in 3.0m 48.25987100601196s\n",
      "\n",
      "Epoch 4/29\n",
      "----------\n",
      "train Loss: 0.5862 Acc: 0.7756\n",
      "val Loss: 0.9043 Acc: 0.7430\n",
      "epochs_val acc: 0.7430\n",
      "epochs_before_best acc: 0.7242\n",
      "epochs_after_best acc: 0.7430\n",
      "Training complete in 3.0m 47.893232345581055s\n",
      "\n",
      "Epoch 5/29\n",
      "----------\n",
      "train Loss: 0.7839 Acc: 0.8127\n",
      "val Loss: 0.6539 Acc: 0.7754\n",
      "epochs_val acc: 0.7754\n",
      "epochs_before_best acc: 0.7430\n",
      "epochs_after_best acc: 0.7754\n",
      "Training complete in 3.0m 48.048932790756226s\n",
      "\n",
      "Epoch 6/29\n",
      "----------\n",
      "train Loss: 0.3203 Acc: 0.8455\n",
      "val Loss: 0.5587 Acc: 0.7755\n",
      "epochs_val acc: 0.7755\n",
      "epochs_before_best acc: 0.7754\n",
      "epochs_after_best acc: 0.7755\n",
      "Training complete in 3.0m 47.62705326080322s\n",
      "\n",
      "Epoch 7/29\n",
      "----------\n",
      "train Loss: 0.4231 Acc: 0.8695\n",
      "val Loss: 0.9052 Acc: 0.7698\n",
      "Training complete in 3.0m 29.393680334091187s\n",
      "\n",
      "Epoch 8/29\n",
      "----------\n",
      "train Loss: 0.4911 Acc: 0.8946\n",
      "val Loss: 1.0168 Acc: 0.7709\n",
      "Training complete in 3.0m 28.98490023612976s\n",
      "\n",
      "Epoch 9/29\n",
      "----------\n",
      "train Loss: 0.2336 Acc: 0.9116\n",
      "val Loss: 0.3996 Acc: 0.7744\n",
      "Training complete in 3.0m 28.839845418930054s\n",
      "\n",
      "Epoch 10/29\n",
      "----------\n",
      "train Loss: 0.2836 Acc: 0.9244\n",
      "val Loss: 1.2052 Acc: 0.7564\n",
      "Training complete in 3.0m 28.80575466156006s\n",
      "\n",
      "Epoch 11/29\n",
      "----------\n",
      "train Loss: 0.0332 Acc: 0.9381\n",
      "val Loss: 1.1357 Acc: 0.7742\n",
      "Training complete in 3.0m 28.90140724182129s\n",
      "...Early Stopping...\n",
      "\n",
      "Best val Acc: 0.7754896313364056\n",
      "FOLD 8\n",
      "==========\n",
      "Epoch 0/29\n",
      "----------\n",
      "train Loss: 2.0967 Acc: 0.4273\n",
      "val Loss: 2.0729 Acc: 0.4955\n",
      "epochs_val acc: 0.4955\n",
      "epochs_before_best acc: 0.0000\n",
      "epochs_after_best acc: 0.4955\n",
      "Training complete in 3.0m 47.189958572387695s\n",
      "\n",
      "Epoch 1/29\n",
      "----------\n",
      "train Loss: 1.4553 Acc: 0.4996\n",
      "val Loss: 2.0653 Acc: 0.5513\n",
      "epochs_val acc: 0.5513\n",
      "epochs_before_best acc: 0.4955\n",
      "epochs_after_best acc: 0.5513\n",
      "Training complete in 3.0m 47.7481427192688s\n",
      "\n",
      "Epoch 2/29\n",
      "----------\n",
      "train Loss: 1.5306 Acc: 0.5622\n",
      "val Loss: 1.5405 Acc: 0.5961\n",
      "epochs_val acc: 0.5961\n",
      "epochs_before_best acc: 0.5513\n",
      "epochs_after_best acc: 0.5961\n",
      "Training complete in 3.0m 48.39624309539795s\n",
      "\n",
      "Epoch 3/29\n",
      "----------\n",
      "train Loss: 0.8322 Acc: 0.6680\n",
      "val Loss: 1.0960 Acc: 0.7124\n",
      "epochs_val acc: 0.7124\n",
      "epochs_before_best acc: 0.5961\n",
      "epochs_after_best acc: 0.7124\n",
      "Training complete in 3.0m 47.503114461898804s\n",
      "\n",
      "Epoch 4/29\n",
      "----------\n",
      "train Loss: 0.5818 Acc: 0.7332\n",
      "val Loss: 0.7781 Acc: 0.7652\n",
      "epochs_val acc: 0.7652\n",
      "epochs_before_best acc: 0.7124\n",
      "epochs_after_best acc: 0.7652\n",
      "Training complete in 3.0m 47.897587299346924s\n",
      "\n",
      "Epoch 5/29\n",
      "----------\n",
      "train Loss: 0.7388 Acc: 0.7802\n",
      "val Loss: 0.7001 Acc: 0.7763\n",
      "epochs_val acc: 0.7763\n",
      "epochs_before_best acc: 0.7652\n",
      "epochs_after_best acc: 0.7763\n",
      "Training complete in 3.0m 47.49499750137329s\n",
      "\n",
      "Epoch 6/29\n",
      "----------\n",
      "train Loss: 0.7048 Acc: 0.8060\n",
      "val Loss: 0.7857 Acc: 0.7606\n",
      "Training complete in 3.0m 28.611326456069946s\n",
      "\n",
      "Epoch 7/29\n",
      "----------\n",
      "train Loss: 0.6879 Acc: 0.8363\n",
      "val Loss: 0.8383 Acc: 0.7786\n",
      "epochs_val acc: 0.7786\n",
      "epochs_before_best acc: 0.7763\n",
      "epochs_after_best acc: 0.7786\n",
      "Training complete in 3.0m 47.25861620903015s\n",
      "\n",
      "Epoch 8/29\n",
      "----------\n",
      "train Loss: 0.5605 Acc: 0.8596\n",
      "val Loss: 1.2123 Acc: 0.7647\n",
      "Training complete in 3.0m 28.80460548400879s\n",
      "\n",
      "Epoch 9/29\n",
      "----------\n",
      "train Loss: 0.3443 Acc: 0.8805\n",
      "val Loss: 1.1756 Acc: 0.7862\n",
      "epochs_val acc: 0.7862\n",
      "epochs_before_best acc: 0.7786\n",
      "epochs_after_best acc: 0.7862\n",
      "Training complete in 3.0m 47.781596660614014s\n",
      "\n",
      "Epoch 10/29\n",
      "----------\n",
      "train Loss: 0.0717 Acc: 0.8971\n",
      "val Loss: 0.5243 Acc: 0.7775\n",
      "Training complete in 3.0m 28.86707901954651s\n",
      "\n",
      "Epoch 11/29\n",
      "----------\n",
      "train Loss: 0.2528 Acc: 0.9173\n",
      "val Loss: 1.0347 Acc: 0.7952\n",
      "epochs_val acc: 0.7952\n",
      "epochs_before_best acc: 0.7862\n",
      "epochs_after_best acc: 0.7952\n",
      "Training complete in 3.0m 48.03945827484131s\n",
      "\n",
      "Epoch 12/29\n",
      "----------\n",
      "train Loss: 0.1042 Acc: 0.9291\n",
      "val Loss: 1.2269 Acc: 0.7763\n",
      "Training complete in 3.0m 28.79344868659973s\n",
      "\n",
      "Epoch 13/29\n",
      "----------\n",
      "train Loss: 0.1375 Acc: 0.9413\n",
      "val Loss: 0.6570 Acc: 0.7898\n",
      "Training complete in 3.0m 28.713958501815796s\n",
      "\n",
      "Epoch 14/29\n",
      "----------\n",
      "train Loss: 0.0178 Acc: 0.9501\n",
      "val Loss: 1.2484 Acc: 0.7864\n",
      "Training complete in 3.0m 29.011558294296265s\n",
      "\n",
      "Epoch 15/29\n",
      "----------\n",
      "train Loss: 0.0466 Acc: 0.9547\n",
      "val Loss: 1.0335 Acc: 0.7918\n",
      "Training complete in 3.0m 28.64634370803833s\n",
      "\n",
      "Epoch 16/29\n",
      "----------\n",
      "train Loss: 0.0998 Acc: 0.9603\n",
      "val Loss: 1.6234 Acc: 0.7818\n",
      "Training complete in 3.0m 29.033082246780396s\n",
      "...Early Stopping...\n",
      "\n",
      "Best val Acc: 0.7951636904761905\n",
      "FOLD 9\n",
      "==========\n",
      "Epoch 0/29\n",
      "----------\n",
      "train Loss: 2.2941 Acc: 0.4280\n",
      "val Loss: 1.8288 Acc: 0.4957\n",
      "epochs_val acc: 0.4957\n",
      "epochs_before_best acc: 0.0000\n",
      "epochs_after_best acc: 0.4957\n",
      "Training complete in 3.0m 48.49427533149719s\n",
      "\n",
      "Epoch 1/29\n",
      "----------\n",
      "train Loss: 1.5640 Acc: 0.5192\n",
      "val Loss: 1.3350 Acc: 0.5650\n",
      "epochs_val acc: 0.5650\n",
      "epochs_before_best acc: 0.4957\n",
      "epochs_after_best acc: 0.5650\n",
      "Training complete in 3.0m 47.95426821708679s\n",
      "\n",
      "Epoch 2/29\n",
      "----------\n",
      "train Loss: 1.2616 Acc: 0.6322\n",
      "val Loss: 1.4030 Acc: 0.6999\n",
      "epochs_val acc: 0.6999\n",
      "epochs_before_best acc: 0.5650\n",
      "epochs_after_best acc: 0.6999\n",
      "Training complete in 3.0m 48.339593172073364s\n",
      "\n",
      "Epoch 3/29\n",
      "----------\n",
      "train Loss: 1.1675 Acc: 0.7234\n",
      "val Loss: 0.5384 Acc: 0.7353\n",
      "epochs_val acc: 0.7353\n",
      "epochs_before_best acc: 0.6999\n",
      "epochs_after_best acc: 0.7353\n",
      "Training complete in 3.0m 47.516239404678345s\n",
      "\n",
      "Epoch 4/29\n",
      "----------\n",
      "train Loss: 0.5569 Acc: 0.7687\n",
      "val Loss: 0.7884 Acc: 0.7629\n",
      "epochs_val acc: 0.7629\n",
      "epochs_before_best acc: 0.7353\n",
      "epochs_after_best acc: 0.7629\n",
      "Training complete in 3.0m 47.80438184738159s\n",
      "\n",
      "Epoch 5/29\n",
      "----------\n",
      "train Loss: 0.4938 Acc: 0.8089\n",
      "val Loss: 1.1039 Acc: 0.7394\n",
      "Training complete in 3.0m 28.804453134536743s\n",
      "\n",
      "Epoch 6/29\n",
      "----------\n",
      "train Loss: 0.4487 Acc: 0.8332\n",
      "val Loss: 0.5926 Acc: 0.7920\n",
      "epochs_val acc: 0.7920\n",
      "epochs_before_best acc: 0.7629\n",
      "epochs_after_best acc: 0.7920\n",
      "Training complete in 3.0m 47.757972717285156s\n",
      "\n",
      "Epoch 7/29\n",
      "----------\n",
      "train Loss: 0.6152 Acc: 0.8539\n",
      "val Loss: 0.2681 Acc: 0.7654\n",
      "Training complete in 3.0m 29.317933082580566s\n",
      "\n",
      "Epoch 8/29\n",
      "----------\n",
      "train Loss: 0.5270 Acc: 0.8759\n",
      "val Loss: 1.0238 Acc: 0.7717\n",
      "Training complete in 3.0m 29.052845001220703s\n",
      "\n",
      "Epoch 9/29\n",
      "----------\n",
      "train Loss: 0.1325 Acc: 0.8973\n",
      "val Loss: 0.8561 Acc: 0.7797\n",
      "Training complete in 3.0m 29.202850103378296s\n",
      "\n",
      "Epoch 10/29\n",
      "----------\n",
      "train Loss: 0.8030 Acc: 0.9097\n",
      "val Loss: 0.4019 Acc: 0.7821\n",
      "Training complete in 3.0m 28.87910032272339s\n",
      "\n",
      "Epoch 11/29\n",
      "----------\n",
      "train Loss: 0.1242 Acc: 0.9310\n",
      "val Loss: 0.9729 Acc: 0.7741\n",
      "Training complete in 3.0m 28.93106722831726s\n",
      "...Early Stopping...\n",
      "\n",
      "Best val Acc: 0.7919642857142858\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "# kfold = KFold(n_splits=num_folds, random_state=0, shuffle=True)\n",
    "kfold = StratifiedKFold(n_splits=num_folds, random_state=0, shuffle=True)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "results = {}\n",
    "\n",
    "for fold, (train_ids, val_ids) in enumerate(kfold.split(dataset, dataset.labels)):\n",
    "    print(f'FOLD {fold}')\n",
    "    print('='*10)\n",
    "    \n",
    "    # Sample elements randomly from a given list of ids, no replacement.\n",
    "    train_subsampler = torch.utils.data.SubsetRandomSampler(train_ids)\n",
    "    val_subsampler = torch.utils.data.SubsetRandomSampler(val_ids)\n",
    "    \n",
    "    # Define data loaders for training and testing data in this fold\n",
    "    train_loader = torch.utils.data.DataLoader(\n",
    "        dataset,\n",
    "        batch_size=batch_size,\n",
    "        sampler=train_subsampler,\n",
    "    )\n",
    "    \n",
    "    val_loader = torch.utils.data.DataLoader(\n",
    "        dataset,\n",
    "        batch_size=batch_size,\n",
    "        sampler=val_subsampler,\n",
    "    )\n",
    "    \n",
    "    # 모델을 불러온다. (huggingface로 계속 불러오면 메모리 초과 발생!)\n",
    "    model.load_state_dict(torch.load(PATH))\n",
    "    \n",
    "    no_decay = ['bias', 'LayerNorm.weight']\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
    "        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
    "    ]\n",
    "    optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate)\n",
    "    \n",
    "    t_total = len(train_loader) * num_epochs\n",
    "    warmup_step = int(t_total * warmup_ratio)\n",
    "\n",
    "    scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=warmup_step, num_training_steps=t_total)\n",
    "    \n",
    "    best_acc = 0.0\n",
    "    early_stopping_count = 0\n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        print(f'Epoch {epoch}/{num_epochs-1}')\n",
    "        print('-' * 10)\n",
    "\n",
    "        train_acc = 0.0\n",
    "        val_acc = 0.0\n",
    "\n",
    "        since = time.time()\n",
    "\n",
    "        #################### Train ####################\n",
    "        train_loss = 0.0\n",
    "\n",
    "        model.train()\n",
    "        for batch_id, batch in enumerate(train_loader):\n",
    "            optimizer.zero_grad()\n",
    "            input_ids = batch['input_ids'].to(device)\n",
    "            attention_mask = batch['attention_mask'].to(device)\n",
    "            labels = batch['labels'].to(device)\n",
    "            outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "            loss = outputs[0]\n",
    "            loss.backward()\n",
    "            torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)\n",
    "            optimizer.step()\n",
    "            scheduler.step()\n",
    "            train_acc += calc_accuracy(outputs[1], labels)\n",
    "            train_loss = loss.data.cpu().numpy()\n",
    "\n",
    "        print(f\"train Loss: {train_loss:.4f} Acc: {train_acc/(batch_id+1):.4f}\")\n",
    "\n",
    "        #################### Validation ####################\n",
    "        val_loss =0.0\n",
    "\n",
    "        model.eval()\n",
    "        for batch_id, batch in enumerate(val_loader):\n",
    "            input_ids = batch['input_ids'].to(device)\n",
    "            attention_mask = batch['attention_mask'].to(device)\n",
    "            labels = batch['labels'].to(device)\n",
    "            outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "            val_acc += calc_accuracy(outputs[1], labels)\n",
    "            val_loss = outputs[0].data.cpu().numpy()\n",
    "\n",
    "        print(f\"val Loss: {val_loss:.4f} Acc: {val_acc/(batch_id+1):.4f}\")\n",
    "\n",
    "        #################### model save ####################\n",
    "        if (val_acc/(batch_id+1)) > best_acc:\n",
    "            early_stopping_count = 0\n",
    "            print(f\"epochs_val acc: {val_acc/(batch_id+1):.4f}\")\n",
    "            print(f\"epochs_before_best acc: {best_acc:.4f}\")\n",
    "            best_acc = (val_acc/(batch_id+1))\n",
    "            print(f\"epochs_after_best acc: {best_acc:.4f}\")\n",
    "            torch.save(model.state_dict(), f\"/opt/ml/model/model_state_dict{fold}.pt\")\n",
    "        \n",
    "        else:\n",
    "            early_stopping_count += 1\n",
    "        \n",
    "        #################### running time check ####################\n",
    "        time_elapsed = time.time() - since\n",
    "        print(f'Training complete in {time_elapsed // 60}m {time_elapsed % 60}s')\n",
    "        \n",
    "        #################### Early Stopping ####################\n",
    "        if early_stopping_count >= stop_count:\n",
    "            print('...Early Stopping...')\n",
    "            print()\n",
    "            break\n",
    "        \n",
    "        print()\n",
    "    \n",
    "    results[fold] = best_acc\n",
    "    print(f'Best val Acc: {best_acc}')\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "07c649d8-79c6-4fb1-a692-f3533b817fa1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K-FOLD CROSS VALIDATION RESULTS FOR 10 FOLDS\n",
      "--------------------------------\n",
      "Fold 0: 0.784418202764977 %\n",
      "Fold 1: 0.785426267281106 %\n",
      "Fold 2: 0.7966589861751151 %\n",
      "Fold 3: 0.7632848502304147 %\n",
      "Fold 4: 0.7731494815668203 %\n",
      "Fold 5: 0.7933107718894009 %\n",
      "Fold 6: 0.7743735599078342 %\n",
      "Fold 7: 0.7754896313364056 %\n",
      "Fold 8: 0.7951636904761905 %\n",
      "Fold 9: 0.7919642857142858 %\n",
      "Average: 0.783323972734255 %\n"
     ]
    }
   ],
   "source": [
    "# Print fold results\n",
    "print(f'K-FOLD CROSS VALIDATION RESULTS FOR {num_folds} FOLDS')\n",
    "print('--------------------------------')\n",
    "sum = 0.0\n",
    "for key, value in results.items():\n",
    "    print(f'Fold {key}: {value} %')\n",
    "    sum += value\n",
    "\n",
    "print(f'Average: {sum/len(results.items())} %')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "675172a2-d74b-4fd9-a7b5-3177d0a25207",
   "metadata": {},
   "source": [
    "## Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ceefea44-6653-4175-9c85-b28a74642fdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset_dir = \"/opt/ml/input/data/test/test.tsv\"\n",
    "test_data = load_data(test_dataset_dir)\n",
    "test_label = test_data['label'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c25d7a87-233d-4863-8cff-91198768c0d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_test = tokenized_dataset(test_data, tokenizer)\n",
    "test_dataset = RE_Dataset(tokenized_test ,test_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "60db2012-ea7e-4836-a600-202054d685bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataloader = torch.utils.data.DataLoader(\n",
    "    test_dataset,\n",
    "    batch_size=25,\n",
    "    shuffle=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "91519a94-102c-4d28-9570-06ff00f2c977",
   "metadata": {},
   "outputs": [],
   "source": [
    "oof_pred = None\n",
    "\n",
    "for i in [1, 2, 5, 8, 9]:\n",
    "    model.load_state_dict(torch.load(f'/opt/ml/model/model_state_dict{i}.pt'))\n",
    "    model.eval()\n",
    "\n",
    "    fold_pred = []\n",
    "    for i, batch in enumerate(test_dataloader):\n",
    "        with torch.no_grad():\n",
    "            outputs = model(\n",
    "                input_ids=batch['input_ids'].to(device),\n",
    "                attention_mask=batch['attention_mask'].to(device),\n",
    "#                 token_type_ids=data['token_type_ids'].to(device)\n",
    "            )\n",
    "            logits = outputs[0]\n",
    "            logits = logits.detach().cpu().numpy()\n",
    "\n",
    "        fold_pred.append(logits)\n",
    "\n",
    "    if oof_pred is None:\n",
    "        oof_pred = np.array(fold_pred) / 5\n",
    "    else:\n",
    "        oof_pred += (np.array(fold_pred) / 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8670b56a-d09e-47c8-9aa5-95ad858e6549",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40, 25, 42)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oof_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3ace4df8-a701-450c-985d-493e48939d88",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = np.argmax(oof_pred, axis=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "59d420f4-51d1-4a39-b899-bbac2fecc124",
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = [pred[i][j] for i in range(len(pred)) for j in range(len(pred[i]))]\n",
    "submission = pd.DataFrame(submission, columns=['pred'])\n",
    "submission.to_csv('/opt/ml/prediction/submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57be7512-f3f0-4a27-9e40-d613d708d132",
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import Audio\n",
    "Audio(\"https://upload.wikimedia.org/wikipedia/commons/0/05/Beep-09.ogg\", autoplay=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4200088-4667-4a4a-aa3a-9419f21d55b7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5811c7f6-7b9f-48d3-a396-23089922be88",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e727f310-6c6f-48c3-a9b0-4c23997f75c4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}