LM_wikitext_MOTAS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data can be downloaded from [here](https://einstein.ai/research/the-wikitext-long-term-dependency-language-modeling-dataset). as suggested by smerity, we only add the eos flag at each end of sentence. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "EOS = '<eos>'\n",
    "PATH=Path('../data/wikitext')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_file(filename):\n",
    "    tokens = []\n",
    "    with open(PATH/filename, encoding='utf8') as f:\n",
    "        for line in f:\n",
    "            tokens.append(line.split() + [EOS])\n",
    "    return np.array(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_tok = read_file('wiki.train.tokens')\n",
    "val_tok = read_file('wiki.valid.tokens')\n",
    "tst_tok = read_file('wiki.test.tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(36718, 3760, 4358)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trn_tok), len(val_tok), len(tst_tok)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we numericalize the tokens."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnt = Counter(word for sent in trn_tok for word in sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('the', 113161),\n",
       " (',', 99913),\n",
       " ('.', 73388),\n",
       " ('of', 56889),\n",
       " ('<unk>', 54625),\n",
       " ('and', 50603),\n",
       " ('in', 39453),\n",
       " ('to', 39190),\n",
       " ('<eos>', 36718),\n",
       " ('a', 34237)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnt.most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "itos = [o for o,c in cnt.most_common()]\n",
    "itos.insert(0,'<pad>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33279"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_size = len(itos); vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "stoi = collections.defaultdict(lambda : 5, {w:i for i,w in enumerate(itos)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_ids = np.array([([stoi[w] for w in s]) for s in trn_tok])\n",
    "val_ids = np.array([([stoi[w] for w in s]) for s in val_tok])\n",
    "tst_ids = np.array([([stoi[w] for w in s]) for s in tst_tok])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is the usual AWD LSTM with three layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "em_sz,nh,nl = 400,1150,3\n",
    "bptt, bs = 70, 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_dl = LanguageModelLoader(np.concatenate(trn_ids), bs, bptt)\n",
    "val_dl = LanguageModelLoader(np.concatenate(val_ids), bs, bptt)\n",
    "md = LanguageModelData(PATH, 0, vocab_size, trn_dl, val_dl, bs=bs, bptt=bptt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "drops_jh = np.array([0.25, 0.1, 0.2, 0.02, 0.15]) #Jeremy's dropouts\n",
    "drops_sm = np.array([0.5,0.4,0.5,0.1,0.3]) #Smerity's dropouts from the paper\n",
    "drops = np.array([0.6,0.4,0.5,0.1,0.2]) #Smerity's dropouts from the github repo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training schedule: 1cycle with either a third phase with cosine annealing or linear decay at one hundreth of the lowest lr. The second one seems to be slightly betters, but by a hair."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_cycle(steps,lr,div,max_mom,min_mom, wd):\n",
    "    return [TrainingPhase(epochs=steps[0], opt_fn=optim.SGD, lr=(lr/div,lr), lr_decay=DecayType.LINEAR, \n",
    "                          momentum=(max_mom,min_mom), momentum_decay=DecayType.LINEAR, wds=wd),\n",
    "           TrainingPhase(epochs=steps[1], opt_fn=optim.SGD, lr=(lr,lr/div), lr_decay=DecayType.LINEAR, \n",
    "                          momentum=(min_mom,max_mom), momentum_decay=DecayType.LINEAR, wds=wd),\n",
    "           TrainingPhase(epochs=steps[2], opt_fn=optim.SGD, lr=lr/div, lr_decay=DecayType.COSINE, \n",
    "                          momentum=max_mom, wds=wd)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_cycle_lin(steps,lr,div,max_mom,min_mom, wd):\n",
    "    return [TrainingPhase(epochs=steps[0], opt_fn=optim.SGD, lr=(lr/div,lr), lr_decay=DecayType.LINEAR, \n",
    "                          momentum=(max_mom,min_mom), momentum_decay=DecayType.LINEAR, wds=wd),\n",
    "           TrainingPhase(epochs=steps[1], opt_fn=optim.SGD, lr=(lr,lr/div), lr_decay=DecayType.LINEAR, \n",
    "                          momentum=(min_mom,max_mom), momentum_decay=DecayType.LINEAR, wds=wd),\n",
    "           TrainingPhase(epochs=steps[2], opt_fn=optim.SGD, lr=(lr/div,lr/(div*100)), lr_decay=DecayType.LINEAR, \n",
    "                          momentum=max_mom, wds=wd)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def custom_cycle(steps, lr, opt_fn, div, max_mom, min_mom, wd):\n",
    "    return [TrainingPhase(steps[0], opt_fn, lr=(lr/div,lr), lr_decay=DecayType.LINEAR, \n",
    "                          momentum=(max_mom,min_mom), momentum_decay=DecayType.LINEAR, wds=wd),\n",
    "            TrainingPhase(steps[1], opt_fn, lr=lr, momentum=min_mom, wds=wd),\n",
    "            TrainingPhase(steps[2], opt_fn, lr=(lr,lr/div), lr_decay=DecayType.LINEAR, \n",
    "                          momentum=(min_mom,max_mom), momentum_decay=DecayType.LINEAR, wds=wd),\n",
    "            TrainingPhase(steps[3], opt_fn, lr=lr/div, lr_decay=DecayType.COSINE, \n",
    "                          momentum=max_mom, wds=wd)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Helper functions for the evaluation of the model at the end. TextReader is rewritten from the LanguageModelLoader class to have a constant bptt and only one batch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextReader():\n",
    "\n",
    "    def __init__(self, nums, bptt, backwards=False):\n",
    "        self.bptt,self.backwards = bptt,backwards\n",
    "        self.data = self.batchify(nums)\n",
    "        self.i,self.iter = 0,0\n",
    "        self.n = len(self.data)\n",
    "\n",
    "    def __iter__(self):\n",
    "        self.i,self.iter = 0,0\n",
    "        while self.i < self.n-1 and self.iter<len(self):\n",
    "            res = self.get_batch(self.i, self.bptt)\n",
    "            self.i += self.bptt\n",
    "            self.iter += 1\n",
    "            yield res\n",
    "\n",
    "    def __len__(self): return self.n // self.bptt \n",
    "\n",
    "    def batchify(self, data):\n",
    "        data = np.array(data)[:,None]\n",
    "        if self.backwards: data=data[::-1]\n",
    "        return T(data)\n",
    "\n",
    "    def get_batch(self, i, seq_len):\n",
    "        source = self.data\n",
    "        seq_len = min(seq_len, len(source) - 1 - i)\n",
    "        return source[i:i+seq_len], source[i+1:i+1+seq_len].view(-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Validation without reinitializing the hidden state."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_validate(learn, source, bptt=2000):\n",
    "    data_source = TextReader(source, bptt)\n",
    "    learn.model.eval()\n",
    "    learn.model.reset()\n",
    "    total_loss = 0.\n",
    "    for inputs, targets in tqdm(data_source):\n",
    "        outputs, raws, outs = learn.model(V(inputs))\n",
    "        total_loss += learn.crit(outputs, targets).detach() * targets.size(0)\n",
    "    mean = total_loss / (bptt * len(data_source))\n",
    "    return mean, np.exp(mean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cache pointer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot(vec, size=vocab_size):\n",
    "    a = torch.zeros(len(vec), size)\n",
    "    for i,v in enumerate(vec):\n",
    "        a[i,v] = 1.\n",
    "    return V(a)\n",
    "\n",
    "def my_cache_pointer(model, source, theta = 0.662, lambd = 0.1279, window=200, bptt=2000):\n",
    "    data_source = TextReader(source, bptt)\n",
    "    model.eval()\n",
    "    model.reset()\n",
    "    total_loss = 0.\n",
    "    targ_history = None\n",
    "    hid_history = None\n",
    "    for inputs, targets in tqdm(data_source):\n",
    "        outputs, raws, outs = model(V(inputs))\n",
    "        p_vocab = F.softmax(outputs,1)\n",
    "        start = 0 if targ_history is None else targ_history.size(0)\n",
    "        targ_history = one_hot(targets) if targ_history is None else torch.cat([targ_history, one_hot(targets)])\n",
    "        hiddens = raws[-1].squeeze() #results of the last layer + remove the batch size.\n",
    "        hid_history = hiddens if hid_history is None else torch.cat([hid_history, hiddens])\n",
    "        for i, pv in enumerate(p_vocab):\n",
    "            #Get the cached values\n",
    "            p = pv\n",
    "            if start + i > 0:\n",
    "                targ_cache = targ_history[:start+i] if start + i <= window else targ_history[start+i-window:start+i]\n",
    "                hid_cache = hid_history[:start+i] if start + i <= window else hid_history[start+i-window:start+i]\n",
    "                all_dot_prods = torch.mv(theta * hid_cache, hiddens[i])\n",
    "                exp_dot_prods = F.softmax(all_dot_prods).unsqueeze(1)\n",
    "                p_cache = (exp_dot_prods.expand_as(targ_cache) * targ_cache).sum(0).squeeze()\n",
    "                p = (1-lambd) * pv + lambd * p_cache\n",
    "            targ_pred = p[targets[i]]\n",
    "            total_loss -= torch.log(targ_pred.detach())\n",
    "        targ_history = targ_history[-window:]\n",
    "        hid_history = hid_history[-window:]\n",
    "    mean = total_loss / (bptt * len(data_source))\n",
    "    return mean, np.exp(mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FakeLinearDecoder(nn.Module):\n",
    "    #This module only applies the dropout to the last ouput but defines all the parameters we will need\n",
    "    #so that they are all registered in the right layer in the learner. Everything will happen in the loss\n",
    "    #as we need the targets.\n",
    "    initrange=0.1\n",
    "    def __init__(self, n_out, nhid, nhid_final, dropout, dropoutl, splits, K=1, tie_encoder=None):\n",
    "        super().__init__()\n",
    "        self.decoder = nn.Linear(nhid, n_out)\n",
    "        #Dropout to apply to the output of the RNN\n",
    "        self.dropout = LockedDropout(dropout)\n",
    "        #Dropout to apply to the latent tensor\n",
    "        self.dropoutl = LockedDropout(dropoutl)\n",
    "        if tie_encoder: self.decoder.weight = tie_encoder.weight\n",
    "        else: self.decoder.weight.data.uniform_(-self.initrange, self.initrange)\n",
    "        #Weights to compute the prior vector\n",
    "        self.prior = nn.Linear(nhid_final, K, bias=False)\n",
    "        #Weights + activation to compute the latent tensor\n",
    "        self.latent = nn.Sequential(nn.Linear(nhid_final, nhid * K), nn.Tanh())\n",
    "        #Additional set of weights for the placeholders (new tokens that indicates the word is in another split)\n",
    "        self.placeholder = nn.Linear(nhid, len(splits))\n",
    "\n",
    "    def forward(self, input):\n",
    "        raw_outputs, outputs = input\n",
    "        output = self.dropout(outputs[-1])\n",
    "        return output, raw_outputs, outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MOTAS(nn.Module):\n",
    "    #Mixture Of Tied Adaptive Softmaxes\n",
    "    \n",
    "    def __init__(self, n_out, nhid, nhid_final, splits, K, prior, latent, decoder, placeholder, dropoutl):\n",
    "        super().__init__()\n",
    "        self.n_out, self.nhid, self.nhid_final, self.K, self.splits = n_out, nhid, nhid_final, K, splits\n",
    "        self.prior, self.latent, self.decoder, self.placeholder = prior, latent, decoder, placeholder\n",
    "        self.dropoutl = dropoutl\n",
    "    \n",
    "    def split_on_target(self, splits, prior, latent, target):\n",
    "        #Splits the prior and latent tensors dependind on where the target is in the splits.\n",
    "        split_idx = torch.zeros_like(target).long()\n",
    "        split_prior, split_latent, split_targ = [], [], []\n",
    "        for split in splits:\n",
    "            mask = (target >= split).long()\n",
    "            split_idx += mask\n",
    "        for idx in range(len(splits)+1):\n",
    "            mask = split_idx == idx\n",
    "            targ_msk = target[mask]\n",
    "            split_targ.append(targ_msk)\n",
    "            prior_msk = prior[mask.unsqueeze(1).expand_as(prior)]\n",
    "            latent_msk = latent[mask.unsqueeze(1).expand_as(latent)]\n",
    "            if targ_msk.size(0) != 0: \n",
    "                prior_msk = prior_msk.view(targ_msk.size(0),-1)\n",
    "                latent_msk = latent_msk.view(targ_msk.size(0),-1)\n",
    "            split_prior.append(prior_msk)\n",
    "            split_latent.append(latent_msk)\n",
    "        return split_prior, split_latent, split_targ\n",
    "\n",
    "    def mix_softmax(self, pri, lat, weight, bias):\n",
    "        #Computes the mix of softmaxes. The latent state goes through the relevenat part of the decoder, then\n",
    "        #we average the predictions with the weights in the prior.\n",
    "        lat = lat.view(-1, self.K, self.nhid)\n",
    "        act = F.linear(lat, weight, bias)\n",
    "        softmaxed = F.softmax(act, dim=2)\n",
    "        probs = (pri.unsqueeze(2) * softmaxed).sum(dim=1)\n",
    "        log_probs = torch.log(probs.add_(1e-8))\n",
    "        return log_probs\n",
    "        \n",
    "    def log_probs(self, split_pri, split_lat):\n",
    "        #First we compute the softmax of the the head + placeholders\n",
    "        head_weight = torch.cat([self.decoder.weight[:self.splits[0]], self.placeholder.weight], dim=0)\n",
    "        head_bias = torch.cat([self.decoder.bias[:self.splits[0]], self.placeholder.bias], dim=0)\n",
    "        all_priors = torch.cat([sp for sp in split_pri if len(sp) != 0], dim=0)\n",
    "        all_latents = torch.cat([sl for sl in split_lat if len(sl) != 0], dim=0)\n",
    "        seps = np.array([so.size(0) for so in split_pri]).cumsum()\n",
    "        head_log_probs = self.mix_softmax(all_priors, all_latents, head_weight, head_bias)\n",
    "        results = [head_log_probs[:seps[0],:self.splits[0]]]\n",
    "        #Then we compute separately the softmax for each split, and the prob we will send is p(placeholder) * softmax\n",
    "        for i, (start, end) in enumerate(zip(self.splits, self.splits[1:] + [self.n_out])):\n",
    "            if split_pri[i+1].size(0) == 0:\n",
    "                results.append(split_out[i+1])\n",
    "                continue\n",
    "            split_log_probs = self.mix_softmax(split_pri[i+1], split_lat[i+1], \n",
    "                                               self.decoder.weight[start:end], self.decoder.bias[start:end])\n",
    "            #log(p(placeholder) * softmax) = log(p(placeholder)) + log(softmax)\n",
    "            results.append(split_log_probs + head_log_probs[seps[i]:seps[i+1],-len(self.splits)+i].unsqueeze(1))\n",
    "        return results\n",
    "    \n",
    "    def forward(self, output, target):\n",
    "        #Get the prior and the latent\n",
    "        priors = F.softmax(self.prior(output.view(-1, self.nhid_final)), dim=1)\n",
    "        latents = self.dropoutl(self.latent(output)).view(-1, self.nhid * self.K)\n",
    "        split_pri, split_lat, split_targ = self.split_on_target(self.splits, priors, latents, target)\n",
    "        log_probs = self.log_probs(split_pri, split_lat)\n",
    "        loss, start_idx = 0, 0\n",
    "        diffs = [0] + self.splits\n",
    "        for diff, log_prob, targ in zip(diffs, log_probs, split_targ):\n",
    "            if targ.size(0) != 0: loss += F.nll_loss(log_prob, targ - diff) * targ.size(0)\n",
    "        return loss / target.size(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_lm_MOTAS(md, opt_fn, n_tok, emb_sz, nhid, nhid_final, nlayers, pad_token, splits, K,\n",
    "                 dropout=0.4, dropouth=0.3, dropouti=0.5, dropoute=0.1, wdrop=0.5, dropoutl=0.2, qrnn=False):\n",
    "    rnn_enc = RNN_Encoder(n_tok, emb_sz, nhid, nlayers, pad_token, nhid_final = nhid_final,\n",
    "                 dropouth=dropouth, dropouti=dropouti, dropoute=dropoute, wdrop=wdrop, qrnn=qrnn)\n",
    "    enc = rnn_enc.encoder\n",
    "    m = SequentialRNN(rnn_enc, FakeLinearDecoder(n_tok, emb_sz, nhid_final, dropout, dropoutl, splits, K, enc))\n",
    "    model = LanguageModel(to_gpu(m))\n",
    "    m = model.model[1]\n",
    "    learner = RNN_Learner(md, model, opt_fn=opt_fn)\n",
    "    learner.crit = MOTAS(n_tok, emb_sz, nhid_final, splits, K, m.prior, m.latent, m.decoder, m.placeholder, m.dropoutl)\n",
    "    return learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.Adam, betas=(0.8,0.99))\n",
    "splits = [2800, 20000]\n",
    "learner = get_lm_MOTAS(md, opt_fn, vocab_size, em_sz, nh, nl, 0, splits, 5,\n",
    "    dropouti=drops[0], dropout=drops[1], wdrop=drops[2], dropoute=drops[3], dropouth=drops[4], dropoutl=0.29)\n",
    "learner.metrics = []\n",
    "learner.clip=0.12\n",
    "learner.unfreeze()\n",
    "learner.reg_fn=partial(seq2seq_reg, alpha=2, beta=1)\n",
    "wd = 1.2e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f22f23f6b0b412a91eeba7b77cb6900",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 260/297 [00:54<00:07,  4.75it/s, loss=nan] "
     ]
    }
   ],
   "source": [
    "learner.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEOCAYAAACEiBAqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl81NW9//HXJztZISEbhFV2ZFECirhgFaVq3bXVW69WW2uttdf2tv3Z9na9vd1se1tb27rVtlauihuIVdEWQSpIAMMSQBAhQBYSlgQSQkhyfn/MgDEmEGC+853MvJ+Pxzwyc+b7nfM538B8cr7n+z3HnHOIiEjsivM7ABER8ZcSgYhIjFMiEBGJcUoEIiIxTolARCTGKRGIiMQ4JQIRkRinRCAiEuOUCEREYpwSgYhIjEvwO4Du6Nu3rxs8eLDfYYiI9CjLly+vdc7lHmu7HpEIBg8eTElJid9hiIj0KGa2tTvb6dSQiEiMUyIQEYlxSgQiIjFOiUBEJMYpEYiIxDglAhGRGOdZIjCzR81sp5mtaVf2PTPbYWbvBB+XeFW/iEhP1tLaxnMrt9NwsMXzurzsETwGzOyk/FfOuYnBx0se1i8i0mMt3FjDPU+W8uamWs/r8iwROOcWAru9+nwRkWg2e/l2ctKS+NioPM/r8mOM4C4zWxU8ddTHh/pFRCLanoZmXivbyZWn9Scx3vuv6XAngt8DpwATgUrgF11taGa3m1mJmZXU1NSEKz4REd/NKa2gubWNaycVhaW+sCYC51y1c67VOdcGPARMOcq2Dzrnip1zxbm5x5wzSUQkasxevp2x/TIZXZgZlvrCmgjMrLDdy6uANV1tKyISi9ZX1bN6R13YegPg4eyjZjYLmA70NbPtwHeB6WY2EXDAFuDzXtUvItITzS7ZTmK8ccXE/mGr07NE4Jy7oZPiR7yqT0Skp9t/sIUnS7Zx0ZgCstOSwlav7iwWEYkQ//d2OfuaWrj93KFhrVeJQEQkAjQ2t/DwoveZOjSHCQN6h7VuJQIRkQhw/z82UVXfxFcuGhH2upUIRER8trF6Hw8t3My1k4qYPDg77PUrEYiI+Mg5x3+9sIa05ATu/fgoX2JQIhAR8dGc0gqWbN7N12eOJCc92ZcYlAhERHxysKWVn728gbH9MvnU5IG+xaFEICLik78tKWfH3gN8Y+Yo4uPMtziUCEREfHCguZUHFrzH1KE5nDO8r6+xKBGIiPhg1tvl1O4/yD0zRmDmX28AlAhERMKu6VArf3jjPc4cms2UIeG/XLQjJQIRkTB7qmQbO/cd5O4LhvsdCqBEICISVgdbWvn9gveYMjibqUNz/A4HUCIQEQmrp0u2U1nXxN0XDPd9bOAwJQIRkTBpbmnj9wve4/SBvZk2LDJ6A6BEICISNs+u2M6OvQciqjcAHiYCM3vUzHaa2UeWozSz/zQzZ2b+XjwrIhImh1rb+N2CTUwoyuK8EZG1DruXPYLHgJkdC81sADADKPewbhGRiPL8yh1s2x15vQHwMBE45xYCuzt561fA1wmsWywiEvVaWtv43T83MbZfJh8bled3OB8R1jECM7sc2OGcKw1nvSIifpq7qoItuxojsjcAHi5e35GZpQLfAi7q5va3A7cDDBzo36x8IiIno7XNcf8/NjGqIIMZo/P9DqdT4ewRnAIMAUrNbAtQBKwws4LONnbOPeicK3bOFefmRtbAiohId81bXcnmmgbuvmA4cT7OMHo0YesROOdWA0dOjgWTQbFzrjZcMYiIhFNbm+P+1zcyIj+dmWM7/Zs3Inh5+egs4C1gpJltN7PbvKpLRCQSzV9Xzcad+7nrY5HbGwAPewTOuRuO8f5gr+oWEYkEDy/azIDsXlw6rtDvUI5KdxaLiHjgnW17WbZlD585a4ivq491hxKBiIgHHnnzfTKSE7h+8gC/QzkmJQIRkRDbsfcAL62u5IYzBpKeHLZrck6YEoGISIj9+V9bALj5rMG+xtFdSgQiIiG0/2ALs5aWc8m4Qvr37uV3ON2iRCAiEkJPLdvGvoMt3Hb2EL9D6TYlAhGREGlpbePRxe8zeXAfJg7o7Xc43aZEICISIq+WVbN9zwFuO3uo36EcFyUCEZEQeXjRZgZmpzJjTGROLtcVJQIRkRBYWb6HFeV7uXXa4Ii/gawjJQIRkRD429JyUpPiuWZSkd+hHDclAhGRk1R34BAvrqrgion9yEhJ9Duc46ZEICJykp5fuYOmQ23cOGWQ36GcECUCEZGT4JzjiaXljOufxbiiLL/DOSFKBCIiJ2FF+V42VO/jxjN67pK6SgQiIifhiaXlpCXFc/mEfn6HcsK8XKHsUTPbaWZr2pX90MxWmdk7ZvaqmfXcIyciMa+uMThIfFp/0nrALKNd8bJH8Bgws0PZz51z451zE4EXge94WL+IiKeeXbmdgy1t3Dil554WAg8TgXNuIbC7Q1l9u5dpgPOqfhERLznnmPV2OROKsji1f88cJD4s7H0ZM/sR8O9AHXB+uOsXEQmF5Vv38G71fn56zTi/QzlpYR8sds59yzk3APgbcFdX25nZ7WZWYmYlNTU14QtQRKQbnlhaTnpyApeN7/lDnX5eNfQEcE1XbzrnHnTOFTvninNzc8MYlojI0e1tbObF1ZVceVq/Hj1IfFhYE4GZDW/38nJgfTjrFxEJhWdX7KC5pefeSdyRZ6nMzGYB04G+ZrYd+C5wiZmNBNqArcAdXtUvIuIF5xxPvF3OxAG9GdMv0+9wQsKzROCcu6GT4ke8qk9EJByWbdnDpp37+dk14/0OJWR0Z7GIyHGY9XY5GckJXDah0O9QQkaJQESkm/Y0NDNvdSVXnd6f1KSeP0h8mBKBiEg3PbNie2CQuAdPMNcZJQIRkW5wzvHksm2cNrA3owqiY5D4MCUCEZFuWLOjno0793PdpAF+hxJySgQiIt3wzIrtJCXEcen46BkkPkyJQETkGJpb2phTWsGMMflk9ep5axIfixKBiMgxvPFuDbsbmrnm9P5+h+IJJQIRkWN4dsV2+qYncc7w6Jz3TIlAROQo9jY28/q6nVw+oT+J8dH5lRmdrRIRCZGXVlfR3NrG1VF6WgiUCEREjmpuaQVDc9MYGyUTzHVGiUBEpAvV9U0seX8Xl0/oh5n5HY5nlAhERLowb1UlzhEVq5AdjRKBiEgX5q6qYExhJsPy0v0OxVNKBCIindi2u5GV5Xu5fGJ09wbAw0RgZo+a2U4zW9Ou7Odmtt7MVpnZc2bW26v6RUROxtxVFQBcOi76ppToyMsewWPAzA5l84FTnXPjgXeBez2sX0TkhM0treT0gb0ZkJ3qdyie8ywROOcWArs7lL3qnGsJvlwCFHlVv4jIidq0cx/rKuu5fEL0nxYCf8cIbgX+7mP9IiKdmlNaSZzBJVE402hnfEkEZvYtoAX421G2ud3MSsyspKamJnzBiUhMc84xt7SCM4fmkJeR4nc4YRH2RGBmNwOXAf/mnHNdbeece9A5V+ycK87Njc6JnkQk8qytqOf92oaYOS0EENbVl81sJvAN4DznXGM46xYR6Y65pRUkxBkzTy3wO5Sw8fLy0VnAW8BIM9tuZrcBvwUygPlm9o6Z/cGr+kVEjldbW+C00LkjcumdmuR3OGHjWY/AOXdDJ8WPeFWfiMjJWlG+h4q6Jr4+c5TfoYSV7iwWEQmaW1pBckIcF47J9zuUsFIiEBEBWlrbmLe6kgtH55OeHNbhU98pEYiIAEs276Z2fzOfmBAb9w60p0QgIgLMKd1BenIC00fm+R1K2CkRiEjMO9jSystrqrhobD4pifF+hxN2SgQiEvMWvVtLfVMLn4ihm8jaUyIQkZg3p7SCPqmJnD2sr9+h+EKJQERiWmNzC/PLqvn4uEIS42PzKzE2Wy0iEvSP9Ts5cKiVT0T5usRH061EYGZfNrNMC3jEzFaY2UVeByci4rU571SQl5HMlCHZfofim+72CG51ztUDFwG5wGeAn3gWlYhIGNQ3HWLBhhouG9+P+DjzOxzfdDcRHD5ClwB/cs6VtisTEemRXllTRXNrW0zeRNZedxPBcjN7lUAieMXMMoA278ISEfHe3FWVDMjuxcQBvf0OxVfdnVDjNmAisNk512hm2QROD4mI9Ei79h9k8aZaPn/uUMxi+wRHd3sEU4ENzrm9ZvZp4NtAnXdhiYh469WyalrbHJfF8NVCh3U3EfweaDSzCcDXga3AXzyLSkTEY/PLqinq04vRhRl+h+K77iaCluD6wlcAv3bO/ZrASmNdMrNHzWynma1pV3adma01szYzKz7xsEVETlxjcwtvbqplxpj8mD8tBN1PBPvM7F7gJmCemcUDicfY5zFgZoeyNcDVwMLjCVJEJJQWvltLc0sbM2JsAZqudDcRfBI4SOB+giqgP/Dzo+3gnFsI7O5Qts45t+FEAhURCZX5ZdVkpiQweXDs3kTWXrcSQfDL/29AlpldBjQ55zRGICI9TktrG/9YX83HRuXF7NxCHXV3ionrgbeB64DrgaVmdq2XgZnZ7WZWYmYlNTU1XlYlIjFkRfle9jQeYsaYAr9DiRjdvY/gW8Bk59xOADPLBV4DZnsVmHPuQeBBgOLiYudVPSISW+aXVZEUH8d5I3P9DiVidLdfFHc4CQTtOo59RUQignOO+WXVnHlKTswtUH803f0yf9nMXjGzW8zsFmAe8NLRdjCzWcBbwEgz225mt5nZVWa2ncANavPM7JWTCV5E5Hhs2rmfLbsadbVQB91Kic65r5nZNcA0ApPNPeice+4Y+9zQxVtH3U9ExCvz11UDMGO0EkF73e4bOeeeAZ7xMBYREU/NL6tmfFEWBVkpfocSUY6aCMxsH9DZQK0BzjmX6UlUIiIhtnNfE+9s28s9F47wO5SIc9RE4JzTJBwiEhX+sW4nzqHxgU7oyh8RiQmHJ5kbVaC/bztSIhCRqKdJ5o5OiUBEot7Cd2s52NKmq4W6oEQgIlHvtXXBSeaGaJK5zigRiEhUa21z/GP9Tk0ydxQ6KiIS1ZZv3cPuhmZNMncUSgQiEtXml1WRGG+cO6Kv36FELCUCEYlahyeZm3pKXzJSjrWoYuxSIhCRqPVejSaZ6w4lAhGJWq+WBSaZu3B0ns+RRDYlAhGJWvPLqhnXP4vCrF5+hxLRlAhEJCodnmROp4WOTYlARKKSJpnrPs8SgZk9amY7zWxNu7JsM5tvZhuDP/t4Vb+IxDZNMtd9XvYIHgNmdij7f8DrzrnhwOvB1yIiIXV4krkLR2uSue7wLBE45xYCuzsUXwH8Ofj8z8CVXtUvIrFr0cbAJHMX6bRQt4R7jCDfOVcJEPzZ5TVdZna7mZWYWUlNTU3YAhSRnm9+mSaZOx4RO1jsnHvQOVfsnCvOzc31OxwR6SE0ydzxC/dRqjazQoDgz51hrl9EotzhSeYu1Gmhbgt3IpgD3Bx8fjPwQpjrF5Eo99q6ahLjjfNG6ExCd3l5+egs4C1gpJltN7PbgJ8AM8xsIzAj+DoiOOf8DkFETpImmTsxCV59sHPuhi7eusCrOjt6atk23txUS0piHMkJ8aQkxlGz7yCVdU3BGKG24SDVdU00NLeSkhhHalICvRLjSU0KPJIT48FBq3O0tjmcc7R2SBodc8hHXncSW0ZKAjlpSeRnpjC2XybjirIYkZdBXJwudRM5Ue/V7Of92gZuPXuI36H0KJ4lgkhQWdfEqu17aTrURlNLK02HWslOTaKoTypmEBcHowoyOG9ELhkpiTQdaqWxuYXG5lYONLfS2BzYJy7OSIqLwwzi44w4Mzp+XX/0UmXr8n3noL7pEBt37ueNd2tobG4FYFBOKrecNZgbzxhIckJ8yI+HSLTTJHMnxnrCKZHi4mJXUlLidxieaG1zvF/bwIryPTy1bBslW/cwILsXX7t4FJeNK1QPQeQ4XP3AYg61OuZ+6Wy/Q4kIZrbcOVd8rO10bZXP4uOMYXnpXF88gKfvmMqfb51CenIid89ayRW/W8yyLR3vyRORztTsO8hKTTJ3QpQIIohZ4EqHeV86m19eP4Fd+w9y3R/e4qtPlVK7/6Df4YlEtNfXVWuSuROkRBCB4uKMq08v4vWvTueL55/CnNIdfOy+Bfz1rS20tkX+qTwRP8wvq6Z/b00ydyKUCCJYr6R4vnbxKP7+5XMZV5TFf72wlit/t5jSbXv9Dk0kohyeZG7GGE0ydyKUCHqAYXnpPH7bGdx/w2lU1zdx5QOL+eZzq9nb2Ox3aCIRQZPMnRwlgh7CzPjEhH68/tXzuHXaEJ5cto0LfvEGCzZolg4RTTJ3cpQIepiMlET+67IxvPils8nNSOaWPy3jJ39fz6HWNr9DE/HF4UnmztckcydMR62HGl2YyfNfnMYNUwbyhzfe46ZHlrKnQaeKJPasKA9MMqerhU6cEkEPlpIYz4+vHscvr5/Aiq17ueqBxWzaud/vsETCan6ZJpk7WUoEUeDq04uYdfsZ7Gtq4aoHFrNooxbykdigSeZCQ4kgSkwalM0Ld02jf+9e3PKnZfzlrS1+hyTiucOTzM3Q3EInRYkgihT1SWX2F85i+ohcvvPCWr7zwhpaNIgsUWx+WeCqOS1Cc3KUCKJMenICD/57MZ8/dyh/eWsrn3lsGXUHDvkdlogn5pdVMa5/FoVZvfwOpUfzJRGY2ZfNbI2ZrTWz//AjhmgWH2fce8lofnbNeJZs3sVVDyxmS22D32GJhJQmmQudsCcCMzsV+BwwBZgAXGZmw8MdRyy4fvIAHr/tDPY0NHPlA4tZsnmX3yGJhMzhSeYuHK1EcLL86BGMBpY45xqdcy3AG8BVPsQRE84YmsPzX5xGTloSNz2ylKeWbfM7JJGQeG1dYJK50YWaZO5k+ZEI1gDnmlmOmaUClwADfIgjZgzKSePZO6dx5tAcvv7MKn780jrNYio9WmNzC4s2apK5UAl7InDOrQN+CswHXgZKgZaO25nZ7WZWYmYlNTW6Lv5kZfVK5E+3TOamMwfxx4WbuePx5TQc/MhhF+kRNMlcaPkyWOyce8Q5d7pz7lxgN7Cxk20edM4VO+eKc3N1x2AoJMTH8cMrT+X7l4/l9XXVXPuHt6jYe8DvsESOmyaZCy2/rhrKC/4cCFwNzPIjjlh181mDefSWyWzf3cgVWt9AehhNMhd6fh3FZ8ysDJgLfNE5t8enOGLW9JF5PHvnWaQkxnH9H9/ixVUVfock0i2aZC70/Do1dI5zboxzboJz7nU/YhAYnp/B83dOY3xRFnc9sZJfzX+XNg0iS4TTJHOhp35VjMtJT+bxz57BtZOK+PXrG7lr1goamzWILJHp8CRzZw7N0SRzIaREICQnxPPza8fzrUtG8/c1VVz5u8VsrtF01hJ5Dk8yp6uFQkuJQIDAUpifO3cof7l1CjX7DnL5bxfz99WVfocl8iEvr6kCYMaYAp8jiS5KBPIh5wzPZd7d5zAsL50v/G0FP5pXpmUwJWK8sraa0wb2piArxe9QoooSgXxEv969eOrzU7l56iAeWvQ+Nz60hMo63W8g/tq+p5HVO+q4eKx6A6GmRCCdSkqI4/tXnMqvPzWRtRX1zPzfRcxbpVNF4p9X11YDKBF4QIlAjuqKif2Zd/c5DO6bxhefWMFXnnyHukatbyDh9/LaKkbmZzCkb5rfoUQdJQI5piF905h9x1TuvmA4L5RWMONXb/BaWbXfYUkMqd1/kJItu7n4VPUGvKBEIN2SGB/HV2aM4IUvTiM7LYnP/qWErzz5Dnsbm/0OTWLAa2XVtDm4eKwuG/WCEoEcl1P7ZzHnrrP58gXDmVNawYxfLeTlNZU4pzuSxTuvrK1iQHYvxhRm+h1KVFIikOOWlBDHPTNG8MJd08hNT+aOx1dw40NLWVtR53doEoX2NR1i8aZdXDymQGsPeESJQE7Y2H5ZzLlrGj+8Yiwbqvdx2f1v8vXZpeysb/I7NIki/9xQQ3NrGzM1PuAZJQI5KQnxcdw0dTD//M/pfO6coTy3cgfT71vAb17fyIHmVr/Dkyjwypoq+qYnc/rAPn6HErWUCCQksnol8s1LRvPaV87jvBG5/HL+u5x/3wIeX7KV5hbdmSwnpulQK//csJOLxuYTF6fTQl5RIpCQGpSTxu8/PYmnPj+Vfr1T+Pbzazj/vgXMertcU1XIcXtzYy2Nza26icxjSgTiiSlDsnnmC2fx51unkJuRzL3Prub8+xbw5LJyDrbolJF0zytrq8hISWDq0By/Q4lqfi1VeY+ZrTWzNWY2y8w0g1QUMgssHvLcnWfxp89MJictiW88s5pzfvpPHliwSXcoy1G1tLbx2rpqLhiVR1KC/mb1UtiPrpn1B+4Gip1zpwLxwKfCHYeEj5lx/sg8nv/iNP5y6xRGFmTws5c3MPUnr/P9uWvZtrvR7xAlAi3cWMOexkN8fFyh36FEvQQf6+1lZoeAVEAL5sYAM+PcEbmcOyKXsop6Hn5zM399ayt//tcWLhlXyM1nDaZ4UB9dKy4AzF6+ney0JM4fmed3KFEv7InAObfDzO4DyoEDwKvOuVfDHYf4a0y/TH55/US+dvFIHvvXFp5YWs6LqyoZkZ/ODVMGcvVpRWSlainCWLWnoZnXynby6TMH6bRQGPhxaqgPcAUwBOgHpJnZpzvZ7nYzKzGzkpqamnCHKWFSmNWLez8+mqXfvICfXTOeXkkJfH9uGVP+5zW++lQpy7fu0fQVMeiFd3bQ3NrGtZOK/A4lJli4/5OZ2XXATOfcbcHX/w6c6Zy7s6t9iouLXUlJSbhCFJ+trajjiaXlvPBOBfsPtjCkbxozTy1g5tgCxhdl6dRRDLjs/kU4B/PuPsfvUHo0M1vunCs+1nZ+jBGUA2eaWSqBU0MXAPqWlyPG9sviR1eN45uXjGZuaQXzVlfy0MLN/H7Be/TLSuGisQXMPLWAyYOziddNRlFnXWU9a3bU891PjPE7lJjhxxjBUjObDawAWoCVwIPhjkMiX1pyAp+aMpBPTRlIXeMhXltXzctrq5j1djmP/WsLvVMTOXNIDtOG5XDWsL4M7Zum3kIUeHLZNhLjjSsm9vc7lJjhy1VDzrnvAt/1o27pmbJSE7lmUhHXTCqi4WALCzbU8Ma7O1m8aRcvr60CoDArhWnD+nL2sL6cNSyHvAzdntLT1Dcd4umSbVwyrpDstCS/w4kZfl0+KnLC0pITuHR8IZeOL8Q5R/nuRhZv2sXiTbW8tq6a2cu3AzAiP51pw/py1il9mTI4W1ch9QBPvr2NhuZWPnv2UL9DiSlKBNKjmRmDctIYlJPGjWcMpK3NUVZZz5ubalm8qZYnlpbzp8VbMINRBZmcMSSbM4ZkM2VINjnpyX6HL+20tLbxp8Xvc8aQbMYVZfkdTkxRIpCoEhdnnNo/i1P7Z3HHeafQdKiVd7bt5e33d7P0/V3837LA+ALA8Lx0zhiazeTB2RQPzqZ/717+Bh/jXlpTRUVdEz+44lS/Q4k5SgQS1VIS4zlzaA5nDs0BhtPc0sbqHXUsfX8XSzfv5rkVO3h8STkQGGM4fVAfxhRmMjI/g5EFGfTv3UvTH4eBc46HF21maN80PjZKdxKHmxKBxJSkhDgmDerDpEF9uHN64HTE+qp9LN+6h2VbdrOyfC/zVlUe2T4tKZ7h+RlHEsPIggxG5GfQNz1JVyiF0LIte1i1vY7/vvJUJV4fKBFITEuIjztyKunmswYDgTVy363ex4aq/bxbvY/1VfW8WlbFkyXbjuyXnZZ0JDmMyM9gZEE6I/IzyEjRgPSJeHjRZvqkJnLN6bqT2A9KBCIdZKQkMmlQNpMGZR8pc85Ru785mCCCj+p9PF0SuMrlsP69ezEiP50RBR/0Ik7JTSclMd6PpvQIW2obmL+umrvOH0avJB0nPygRiHSDmZGbkUxuRjLThvU9Ut7W5tix90AgQbRLEm9uquVQa2D6ljgLLOWZn5nC6MJMRhVkMGFAb8b1zyItWf8F//e1d0lOiOOmqYP8DiVm6V+hyEmIizMGZKcyIDuVC0bnHyk/1NrG1l0NR04v7Wo4yPY9B1iyeRfPrdwR2NdgWF46owoyGVmQwejCDEYXZlKQmRIz4w9rdtTx/DsV3Dn9FN0A6CMlAhEPJMbHMSwvg2F5GVzKhxdW2d3QTOm2vaws38PainqWb93DnNIPluTok5rImH6ZjC7IZEy/TEYVZHJKXhrJCdF12qStzfGDuWX0SU3kjumn+B1OTFMiEAmz7LQkzh+Vx/ntLpOsO3CIDVWBgemyinrKKuv565KtHGxpAyA+zhjSN42R+RkUZqWQn5lCXmYyA7JTGZGfQXoPPMU0e/l23t6ym59eM45MDbL7quf96xGJQlm9EpkSvOP5sJbWNt6vbWB9cNxhfdU+1lbU8fr6apoOtX1o/0E5qYwuyGR0YSajCzMYVZBJUZ/IvQeiYu8B/nteGVMGZ3N98QC/w4l5SgQiESohPo7h+RkMz8/gExM+KHfOUd/Uws76JrbsamR9ZT3rqupZV7mPV8qqOLzESK/EeIblpTMsL53h+emMLshkVGGG72MQLa1tfPWpUlraHD+/bnzMjIdEMiUCkR7GzMjqlUhWr0SG52cwY8wHg9QNB1vYUL2PjdWBHsSmnfs/NEAN0Ds1kRF5GRRl96KoTyppSfH0SUuiMCvlyGknr+6HaGtzfH32Kt7avIufXzueQTlpntQjx0eJQCSKpCUncPrAPpw+sM+HyuubAmMQ6yrrWVdZz3s1Dbz13i6q6nfQ2SKF6ckJFLRLDLkZyeSmBy6f7ZOaREZKAoW9U+ibltyt00/7mg6xu6GZH7+0npfXVvHVGSO4TqeEIoYSgUgMyExJZPLgwAR77bW0ttHU0sbu/c1U1h2gqr6JqromKusCP6vqm9hYXUvt/oO0tH00YyTFx1GQlUKf1ETo5BRPa1sbexoOsWPvASBwyey3Lx3NbWcP8aahckLCngjMbCTwZLuiocB3nHP/G+5YRGJdQnwc6fFxpCcnMDAntcvt2tocdQcOUbP/IHsamqlvaqGq7gA79jZRWXeAvY2HOt0vzmB4XgbD8tLJy0hmZEEG44t6e9UcOUF+LFW5AZgIYGbxwA7guXDHISLdFxdn9ElLoo82oHNZAAAJGElEQVRWDYtKcT7XfwHwnnNuq89xiIjELL8TwaeAWT7HICIS03xLBGaWBFwOPN3F+7ebWYmZldTU1IQ3OBGRGOJnj+DjwArnXHVnbzrnHnTOFTvninNzc8McmohI7PAzEdyATguJiPjOl0RgZqnADOBZP+oXEZEP+HJDmXOuEcjxo24REfkwv68aEhERn5nrbKKRCGNmdcDG4Mu+QG0IPjYLqAvRtl2931l5x7KjvW7/XO0+OWr3yW+rdh+7PNLaPcg5d+yrbZxzEf8AHmz3vCTUn3my23b1fmflHcuO9lrtVrvVbrXbq3a3f/SUU0Nzff7MY23b1fudlXcsO9prtTt01O6T31btPnZ5T2r3ET3i1FB7ZlbinCv2O45wU7tji9odW/xud0/pEbT3oN8B+ETtji1qd2zxtd09rkcgIiKh1RN7BCIiEkJKBCIiMU6JQEQkxkVNIjCz6Wa2yMz+YGbT/Y4nnMwszcyWm9llfscSLmY2Ovi7nm1mX/A7nnAxsyvN7CEze8HMLvI7nnAxs6Fm9oiZzfY7Fq8F/z//Ofh7/rdw1BkRicDMHjWznWa2pkP5TDPbYGabzOz/HeNjHLAfSAG2exVrKIWo3QDfAJ7yJsrQC0W7nXPrnHN3ANcDPeJywxC1+3nn3OeAW4BPehhuyISo3Zudc7d5G6l3jvMYXA3MDv6eLw9LgKG4my0Ed8OdC5wOrGlXFg+8R2Bx+ySgFBgDjANe7PDIA+KC++UDf/O7TWFs94UEVnq7BbjM7zaFq93BfS4H/gXc6Hebwtnu4H6/AE73u00+tHu23+0JwzG4F5gY3OaJcMTny+yjHTnnFprZ4A7FU4BNzrnNAGb2f8AVzrkfA0c7BbIHSPYizlALRbvN7HwgjcA/oANm9pJzrs3TwE9SqH7fzrk5wBwzmwc84V3EoRGi37cBPwH+7pxb4W3EoRHi/9890vEcAwJnNIqAdwjTWZuISARd6A9sa/d6O3BGVxub2dXAxUBv4Lfehuap42q3c+5bAGZ2C1Ab6UngKI739z2dQBc6GXjJ08i8dVztBr5EoBeYZWbDnHN/8DI4Dx3v7zsH+BFwmpndG0wYPV1Xx+A3wG/N7FK8mYbiIyI5EVgnZV3e/eace5boWOjmuNp9ZAPnHgt9KGF1vL/vBcACr4IJo+Nt928IfFH0dMfb7l3AHd6F44tOj4FzrgH4TDgDiYjB4i5sBwa0e10EVPgUSzip3QFqd3SL1Xa3FzHHIJITwTJguJkNMbMkAgOic3yOKRzUbrVb7Y4NEXMMIiIRmNks4C1gpJltN7PbnHMtwF3AK8A64Cnn3Fo/4ww1tVvtVrujt93tRfox0KRzIiIxLiJ6BCIi4h8lAhGRGKdEICIS45QIRERinBKBiEiMUyIQEYlxSgQScma2Pwx1XN7NKbpDWed0MzvrBPY7zcweDj6/xcwiYi4sMxvccVrkTrbJNbOXwxWT+EOJQCKWmcV39Z5zbo5z7ice1Hm0+bemA8edCIBvAvefUEA+c87VAJVmNs3vWMQ7SgTiKTP7mpktM7NVZvb9duXPW2BVtbVmdnu78v1m9gMzWwpMNbMtZvZ9M1thZqvNbFRwuyN/WZvZY2b2GzP7l5ltNrNrg+VxZvZAsI4Xzeylw+91iHGBmf2Pmb0BfNnMPmFmS81spZm9Zmb5wSmE7wDuMbN3zOyc4F/LzwTbt6yzL0szywDGO+dKO3lvkJm9Hjw2r5vZwGD5KWa2JPiZP+ish2WBVazmmVmpma0xs08GyycHj0Opmb1tZhnBv/wXBY/his56NWYWb2Y/b/e7+ny7t58HwrJSlvjE7wUb9Ii+B7A/+PMi4EECsyzGEVhk5Nzge9nBn72ANUBO8LUDrm/3WVuALwWf3wk8HHx+C/Db4PPHgKeDdYwhMMc7wLUEpqiOAwoIrFVxbSfxLgAeaPe6Dx/cdf9Z4BfB598D/rPddk8AZwefDwTWdfLZ5wPPtHvdPu65wM3B57cCzwefvwjcEHx+x+Hj2eFzrwEeavc6i8DiJpuBycGyTAIzDKcCKcGy4UBJ8PlgggulALcD3w4+TwZKgCHB1/2B1X7/u9LDu0ckT0MtPd9FwcfK4Ot0Al9EC4G7zeyqYPmAYPkuoBV4psPnHJ5efDmBNQg687wLrMVQZmb5wbKzgaeD5VVm9s+jxPpku+dFwJNmVkjgy/X9Lva5EBhjdmQ24Uwzy3DO7Wu3TSFQ08X+U9u156/Az9qVXxl8/gRwXyf7rgbuM7OfAi865xaZ2Tig0jm3DMA5Vw+B3gOB+e0nEji+Izr5vIuA8e16TFkEfifvAzuBfl20QaKAEoF4yYAfO+f++KHCwKIyFwJTnXONZraAwFrTAE3OudYOn3Mw+LOVrv/NHmz33Dr87I6Gds/vB37pnJsTjPV7XewTR6ANB47yuQf4oG3H0u2Jv5xz75rZJOAS4Mdm9iqBUzidfcY9QDUwIRhzUyfbGIGe1yudvJdCoB0SpTRGIF56BbjVzNIBzKy/meUR+GtzTzAJjALO9Kj+N4FrgmMF+QQGe7sjC9gRfH5zu/J9QEa7168SmD0SgOBf3B2tA4Z1Uc+/CEw9DIFz8G8Gny8hcOqHdu9/iJn1Axqdc48T6DGcDqwH+pnZ5OA2GcHB7ywCPYU24CYCa+V29ArwBTNLDO47ItiTgEAP4qhXF0nPpkQgnnHOvUrg1MZbZrYamE3gi/RlIMHMVgE/JPDF54VnCCz+sQb4I7AUqOvGft8DnjazRUBtu/K5wFWHB4uBu4Hi4OBqGZ2soOWcW09gWcmMju8F9/9M8DjcBHw5WP4fwFfM7G0Cp5Y6i3kc8LaZvQN8C/hv51wz8EngfjMrBeYT+Gv+AeBmM1tC4Eu9oZPPexgoA1YELyn9Ix/0vs4H5nWyj0QJTUMtUc3M0p1z+y2w5u3bwDTnXFWYY7gH2Oece7ib26cCB5xzzsw+RWDg+ApPgzx6PAsJLCy/x68YxFsaI5Bo96KZ9SYw6PvDcCeBoN8D1x3H9pMIDO4asJfAFUW+MLNcAuMlSgJRTD0CEZEYpzECEZEYp0QgIhLjlAhERGKcEoGISIxTIhARiXFKBCIiMe7/A+thmKXg6x4HAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SetL2WdReg(Callback):\n",
    "    \n",
    "    def __init__(self, learn, l2_reg, wd):\n",
    "        self.learn, self.l2_reg, self.wd = learn, l2_reg, wd\n",
    "    \n",
    "    def on_batch_begin(self):\n",
    "        for group in self.learn.sched.layer_opt.opt_params():\n",
    "            group['wd'] = self.wd\n",
    "            group['weight decay'] = self.l2_reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "wd_reg = SetL2WdReg(learner, wd, wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43cd88dd932d46ab9a85401558f39e9d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=90), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                              \n",
      "    0      6.544169   6.115795  \n",
      "    1      5.925789   5.56642                               \n",
      "    2      5.535354   5.285057                              \n",
      "    3      5.289222   5.07708                               \n",
      "    4      5.129836   4.94611                               \n",
      "    5      5.020651   4.858513                              \n",
      "    6      4.919832   4.81441                               \n",
      "    7      4.854645   4.761397                              \n",
      "    8      4.807414   4.712801                              \n",
      "    9      4.732497   4.676872                              \n",
      "    10     4.705681   4.644085                              \n",
      "    11     4.652177   4.617357                              \n",
      "    12     4.612724   4.599069                              \n",
      "    13     4.582318   4.591534                              \n",
      "    14     4.551856   4.567731                              \n",
      "    15     4.537996   4.558168                              \n",
      "    16     4.508961   4.540078                              \n",
      "    17     4.480189   4.528441                              \n",
      "    18     4.484512   4.515267                              \n",
      "    19     4.464609   4.506759                              \n",
      "    20     4.443874   4.500617                              \n",
      "    21     4.416059   4.488088                              \n",
      "    22     4.413335   4.485022                              \n",
      "    23     4.395638   4.479794                              \n",
      "    24     4.397788   4.472379                              \n",
      "    25     4.396242   4.471117                              \n",
      "    26     4.375485   4.466469                              \n",
      "    27     4.374449   4.457336                              \n",
      "    28     4.371978   4.445516                              \n",
      "    29     4.346598   4.448524                              \n",
      "    30     4.34316    4.448463                              \n",
      "    31     4.337553   4.449999                              \n",
      "    32     4.333152   4.445481                              \n",
      "    33     4.316958   4.438644                              \n",
      "    34     4.329293   4.439793                              \n",
      "    35     4.30839    4.438686                              \n",
      "    36     4.302938   4.442152                              \n",
      "    37     4.296552   4.428961                              \n",
      "    38     4.295244   4.428639                              \n",
      "    39     4.290725   4.428075                              \n",
      "    40     4.28913    4.425407                              \n",
      "    41     4.316573   4.415504                              \n",
      "    42     4.29511    4.40958                               \n",
      "    43     4.285464   4.412889                              \n",
      "    44     4.263164   4.418375                              \n",
      "    45     4.27713    4.404263                              \n",
      "    46     4.26935    4.394091                              \n",
      "    47     4.250396   4.398704                              \n",
      "    48     4.233981   4.390161                              \n",
      "    49     4.216373   4.381469                              \n",
      "    50     4.200906   4.386394                              \n",
      "    51     4.192839   4.384114                              \n",
      "    52     4.18724    4.366968                              \n",
      "    53     4.158387   4.373571                              \n",
      "    54     4.135405   4.367085                              \n",
      "    55     4.136515   4.364371                              \n",
      "    56     4.107619   4.362427                              \n",
      "    57     4.150502   4.341741                              \n",
      "    58     4.08963    4.35162                               \n",
      "    59     4.098088   4.340048                              \n",
      "    60     4.061222   4.340156                              \n",
      "    61     4.077117   4.326524                              \n",
      "    62     4.054523   4.325469                              \n",
      "    63     4.040216   4.322683                              \n",
      "    64     4.000035   4.321453                              \n",
      "    65     3.996018   4.324663                              \n",
      "    66     3.971287   4.323685                              \n",
      "    67     4.035799   4.296824                              \n",
      "    68     3.948798   4.317894                              \n",
      "    69     3.974287   4.301639                              \n",
      "    70     3.936507   4.303917                              \n",
      "    71     3.917754   4.296122                              \n",
      "    72     3.921305   4.294878                              \n",
      "    73     3.879696   4.294697                              \n",
      "    74     3.870561   4.29096                               \n",
      "    75     3.842761   4.284267                              \n",
      "    76     3.851997   4.282599                              \n",
      "    77     3.845998   4.2765                                \n",
      "    78     3.802912   4.273066                              \n",
      "    79     3.81926    4.261469                              \n",
      "    80     3.788018   4.268839                              \n",
      "    81     3.741604   4.266272                              \n",
      "    82     3.741735   4.260792                              \n",
      "    83     3.724474   4.260239                              \n",
      "    84     3.703256   4.260482                              \n",
      "    85     3.718199   4.254985                              \n",
      "    86     3.691324   4.254282                              \n",
      "    87     3.742421   4.250655                              \n",
      "    88     3.686974   4.252103                              \n",
      "    89     3.702364   4.249691                              \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[4.24969143434421]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit_opt_sched(custom_cycle([7.5,37.5,37.5,7.5], 5e-3, opt_fn, 10, 0.8, 0.7, wd), callbacks=[wd_reg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 21%|██▏       | 23/108 [00:06<00:23,  3.64it/s]\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "cuda runtime error (2) : out of memory at /opt/conda/conda-bld/pytorch_1524590031827/work/aten/src/THC/generic/THCStorage.cu:58",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-33-5ca169160391>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmy_validate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-32-347aae5f2590>\u001b[0m in \u001b[0;36mmy_validate\u001b[0;34m(learn, source, bptt)\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_source\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mraws\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0mtotal_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcrit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0mmean\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtotal_loss\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mbptt\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_source\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mmean\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    489\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 491\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    492\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    493\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-23-d173f2bc9667>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, output, target)\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0mlatents\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropoutl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlatent\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnhid\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mK\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0msplit_pri\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msplit_lat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msplit_targ\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit_on_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpriors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlatents\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m         \u001b[0mlog_probs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_probs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msplit_pri\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msplit_lat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m         \u001b[0mdiffs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplits\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-23-d173f2bc9667>\u001b[0m in \u001b[0;36mlog_probs\u001b[0;34m(self, split_pri, split_lat)\u001b[0m\n\u001b[1;32m     53\u001b[0m                 \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m             split_log_probs = self.mix_softmax(split_pri[i+1], split_lat[i+1], \n\u001b[0;32m---> 55\u001b[0;31m                                                self.decoder.weight[start:end], self.decoder.bias[start:end])\n\u001b[0m\u001b[1;32m     56\u001b[0m             \u001b[0;31m#log(p(placeholder) * softmax) = log(p(placeholder)) + log(softmax)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m             \u001b[0mresults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msplit_log_probs\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mhead_log_probs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mseps\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mseps\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplits\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-23-d173f2bc9667>\u001b[0m in \u001b[0;36mmix_softmax\u001b[0;34m(self, pri, lat, weight, bias)\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0mlat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mK\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnhid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m         \u001b[0mact\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m         \u001b[0msoftmaxed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mact\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     36\u001b[0m         \u001b[0mprobs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mpri\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0msoftmaxed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m         \u001b[0mlog_probs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1e-8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36msoftmax\u001b[0;34m(input, dim, _stacklevel)\u001b[0m\n\u001b[1;32m    860\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mdim\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    861\u001b[0m         \u001b[0mdim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_get_softmax_dim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'softmax'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_stacklevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 862\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_nn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    863\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    864\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: cuda runtime error (2) : out of memory at /opt/conda/conda-bld/pytorch_1524590031827/work/aten/src/THC/generic/THCStorage.cu:58"
     ]
    }
   ],
   "source": [
    "my_validate(learner, np.concatenate(val_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.Adam, betas=(0.8,0.99))\n",
    "splits = [2800, 20000]\n",
    "learner = get_lm_MOTAS(md, opt_fn, vocab_size, em_sz, nh, nl, 0, splits, 15,\n",
    "    dropouti=drops[0], dropout=drops[1], wdrop=drops[2], dropoute=drops[3], dropouth=drops[4], dropoutl=0.29)\n",
    "learner.metrics = []\n",
    "learner.clip=0.12\n",
    "learner.unfreeze()\n",
    "learner.reg_fn=partial(seq2seq_reg, alpha=2, beta=1)\n",
    "wd = 1.2e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c251d57198f54d4689af7897632fa827",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 515/595 [01:48<00:16,  4.76it/s, loss=nan] "
     ]
    }
   ],
   "source": [
    "learner.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEOCAYAAACEiBAqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd8HNW99/HPb9WrZVmyLffesA3GMs3UUEIIoYVACBBaAiQh5CY35aY89ybh3oQnIT0PAYeWZpJgQhJMKKEaAja44N67LNmSrS5Zbfc8f+zKCF3Jlm3Nzpbv+/Xal3ZnZ3d+R2vvVzNn5hxzziEiIskr4HcBIiLiLwWBiEiSUxCIiCQ5BYGISJJTEIiIJDkFgYhIklMQiIgkOQWBiEiSUxCIiCQ5BYGISJJL9buAvigqKnJjxozxuwwRkbiybNmy/c654iOtFxdBMGbMGJYuXep3GSIiccXMdvZlPR0aEhFJcgoCEZEkpyAQEUlyCgIRkSSnIBARSXKeBYGZPWJmlWa2ptvyz5vZRjNba2Y/8Gr7IiLSN17uETwGXNx1gZmdB1wOzHTOnQDc5+H2RUTiVmtHkL+9u4e6g+2eb8uzIHDOLQKquy3+DHCvc641sk6lV9sXEYln72yv4Qt/fJelO7p/jfa/aPcRTALOMrMlZvaamc2J8vZFROLCKxsrSU8NcPr4QZ5vK9pXFqcCA4HTgDnAn81snHPOdV/RzG4HbgcYNWpUVIsUEfGTc46X1u/j1LGFZKd7/zUd7T2CMuAvLuxtIAQU9bSic26ec67UOVdaXHzEoTJERBLG2vJ6dhxo5pIZJVHZXrSD4K/ABwDMbBKQDuyPcg0iIjHt6VXlpAaMi08YGpXtebbPYWaPA+cCRWZWBvwX8AjwSOSU0jbgpp4OC4mIJCvnHAtXVnDmxCIG5qRHZZueBYFz7rpenrrBq22KiMS7Fbtr2VN7kC9eOClq29SVxSIiMWThygrSUwJcdMKQqG1TQSAiEiNCIcczq8s5Z3Ix+ZlpUduugkBEJEa8s6OaffWtXDozOmcLdVIQiIjEiIWrKshMC3DB1OgdFgIFgYhITOgIhnh2TQXnTxlCTkZ0r/VVEIiIxIDF26rZ39gW9cNCoCAQEYkJC1eVk5OewnlTBkd92woCERGftXWEeG7tXi6cNoTMtJSob19BICLis39t2U9tczuXzhzmy/YVBCIiPlu4qoL8zFTOmtTjGJyeUxCIiPgoGHK8vGEf508dQkZq9A8LgYJARMRX7+6uoaa5nfOnRr+TuJOCQETERy+tryQ1YJw10b95VxQEIiI+enlDJXPGFDIgK3pjC3WnIBAR8UlZTTMb9jb4elgIFAQiIr55ZUMlAB/w4SKyrhQEIiI+eWlDJWOLchhXnOtrHQoCEREfNLd18ObWA77vDYCCQETEF//acoC2jpCCQEQkWb28YR+5GanMGVPodykKAhGRaHPO8dL6Ss6eVER6qv9fw/5XICKSZNaW11PZ0MoHpkR3JrLeKAhERKLspfWVmMG5k/27mrgrBYGISJS9vGEfJ40soCg3w+9SAAWBiEhUVTa0sLKsjvNj4GyhTgoCEZEoenVDFUDM9A+AgkBEJKpe2VjJ0PxMppbk+V3KIQoCEZEoaQ+GeGPzfs6dXIyZ+V3OIQoCEZEoeXd3LQ2tHZwzKTbOFuqkIBARiZLXNlaREjDOmODP3MS9URCIiETJa5uqmDWywNdJaHqiIBARiYLK+hZW76njvBg6bbSTgkBEJApe3RQ+bfS8yUkUBGb2iJlVmtmaHp77spk5M4utA2UiIh55ZUPsnTbaycs9gseAi7svNLORwIXALg+3LSISM9qDIV7fvJ/zpsTWaaOdPAsC59wioLqHp34CfBVwXm1bRCSWrN5TR2NrB2dOiK3TRjtFtY/AzC4D9jjnVkZzuyIiflq87QAAp47zfxKanqRGa0Nmlg18E7ioj+vfDtwOMGrUKA8rExHx1pJt1UwcnBszo412F809gvHAWGClme0ARgDLzWxoTys75+Y550qdc6XFxbG5OyUiciTBkGP5zhpKY2BKyt5EbY/AObcaOHTeVCQMSp1z+6NVg4hItG3a10BDawdzxgz0u5ReeXn66OPAW8BkMyszs9u82paISKxaurMGgNLRSbhH4Jy77gjPj/Fq2yIisWLZjmoG52UwsjDL71J6pSuLRUQ8tHRnDaVjBsbk9QOdFAQiIh6prG+hrOYgJ4+K3f4BUBCIiHhm+a5aAGYpCEREktOKXTWkpwSYPjzf71IOS0EgIuKR5btqmDYsn4zUFL9LOSwFgYiIB9qDIVaV1cV8/wAoCEREPLFxbwOtHSFOGlXgdylHpCAQEfHAqrI6AE4cMcDnSo5MQSAi4oHVe2oZkJXGqMJsv0s5IgWBiIgHVu6uY+aIATF9IVknBYGISD9raQ+yaV8DM4bH/mEhUBCIiPS79RX1dIQcM0fEfkcxKAhERPpdZ0fxzDjoKAYFgYhIv1tVVkdRbgYlAzL9LqVPFAQiIv1s9Z7auOkoBgWBiEi/amrtYEtlY9x0FIOCQESkX60tryfk4MSRCgIRkaS0qiw89PR07RGIiCSnVWV1lAzIZHBefHQUg4JARKRfrd5TFzenjXZSEIiI9JO6g+1s398UNxeSdVIQiIj0kzV7wheSxdMZQ6AgEBHpN/F2RXEnBYGISD9ZvaeWUYXZFGSn+13KUVEQiIj0k5W765gRZ3sDoCAQEekXBxpb2VN7MC5mJOtOQSAi0g9WH+oojq8zhkBBICLSL1aV1WEG04fn+13KUVMQiIj0g1VldYwryiEvM83vUo6agkBEpB+Eh56Ov8NCoCAQETlu++pb2FffGncXknVSEIiIHKfOC8niaejprhQEIiLHaXVZLQGDaSUKgvcxs0fMrNLM1nRZ9kMz22Bmq8zsKTOLzwNqIiJdrCyrY9KQPLLSU/wu5Zh4uUfwGHBxt2X/BKY752YCm4Cve7h9ERHPOeficujprjwLAufcIqC627IXnHMdkYeLgRFebV9EJBr21B6kuqmNGXF6xhD420dwK/Csj9sXETluh0YcjdMzhsCnIDCzbwIdwB8Os87tZrbUzJZWVVVFrzgRkaOwqqyOtBRjSkme36Ucs6gHgZndBFwKXO+cc72t55yb55wrdc6VFhcXR69AEZGjsHpPLVOG5pORGp8dxRDlIDCzi4GvAZc555qjuW0Rkf7mnGNteX1cji/UlZenjz4OvAVMNrMyM7sN+CWQB/zTzN41swe82r6IiNcqG1qpbW5nytD4DoJUr97YOXddD4sf9mp7IiLRtr6iHoApQ+O3fwB0ZbGIyDHbsLcBIO73CPoUBGb2BTPLt7CHzWy5mV3kdXEiIrFs494GSgZkMiA7/oae7qqvewS3OufqgYuAYuAW4F7PqhIRiQPrK+qZHOeHhaDvQWCRn5cAjzrnVnZZJiKSdNqDIbZWNcb9YSHoexAsM7MXCAfB82aWB4S8K0tEJLZtqWykPeiYGscXknXq61lDtwEnAducc81mVkj48JCISFLqPGNoakny7BGcDmx0ztWa2Q3At4A678oSEYlt6yvqSU8NMK4ox+9Sjltfg+BXQLOZnQh8FdgJ/NazqkREYtyGvQ1MGpJLakr8n4Xf1xZ0RMYFuhz4mXPuZ4SvEBYRSUrrK+qZmgAdxdD3PoIGM/s6cCNwlpmlAPF94qyIyDGqbGhhf2NbQvQPQN/3CK4FWglfT7AXGA780LOqRERi2PqK8BXFSRUEkS//PwADzOxSoMU5pz4CEUlKnWcMTUumIDCza4C3gY8B1wBLzOxqLwsTEYlV6yvqGZYAQ0t06msfwTeBOc65SgAzKwZeBBZ4VZiISKxaX1GfMIeFoO99BIHOEIg4cBSvFRFJGC3tQbZWNcX11JTd9XWP4Dkzex54PPL4WuAf3pQkIhK7tlQ2Egy5hNoj6FMQOOe+YmYfBeYSHmxunnPuKU8rExGJQesSaGiJTn2eocw59yTwpIe1iIjEvPUV9WSmBRgzKP6Hluh02CAwswbA9fQU4JxziROJIiJ9EJ6DIJ+UQOKMxH/YIHDOJU5viIjIcXLOsb6igUtmDPW7lH6lM39ERPqooq6FuoPtCdU/AAoCEZE+S6Q5CLpSEIiI9FFnEExJgHmKu1IQiIj00fqKBkYWZpGXmRhDS3RSEIiI9FEizUHQlYJARKQPmts62H6gKeH6B0BBICLSJxv3NuBc4nUUg4JARKRP1iXYHARdKQhERPpgbXk9eZmpjCzM8ruUfqcgEBHpg7Xl9ZwwLB+zxBlaopOCQETkCDqCITZU1HPCsAF+l+IJBYGIyBFsrWqitSPE9OGJ1z8ACgIRkSNaW14HoD2Co2Vmj5hZpZmt6bKs0Mz+aWabIz8HerV9EZH+sra8nozUAOOKEmcOgq683CN4DLi427L/AF5yzk0EXoo8FhGJaWvL65hakk9qSmIeRPGsVc65RUB1t8WXA7+J3P8NcIVX2xcR6Q/OuUNnDCWqaMfbEOdcBUDk5+DeVjSz281sqZktraqqilqBIiJd7apupqGlI2H7ByCGO4udc/Occ6XOudLi4mK/yxGRJPX29vCBjdmjE7dLs8+T1/eTfWZW4pyrMLMSoDIaG3XO0dIeYseBJlIDxuC8TLLSUwgYmNmhn51aO4IcaGwjPTVAWkqA+oPttLQHCa9i5GSkUJSbQVqCHi8Ukfcs2V7NwOw0Jg7O9bsUz0Q7CP4O3ATcG/n5Ny83dt/zG/nNmztoaO044roBg4zUFNJTAzS2dhAMucOubwaF2ekU52VQnJfB4LxMBudnMK4ohxEDs5k1qoDMtJT+aoqI+GTFrhpmjy4kkECT1XfnWRCY2ePAuUCRmZUB/0U4AP5sZrcBu4CPebV9gPGDc7hi1nAG5qSTkRpgVGE2Dqisb6GlPUjIgXMQco5gyNEWDNHaHiQ/K42SAVl0hEK0dYTIz0ojKy0FR3jvorG1g8r6ViobWqlqaKWqoYWtlY1UNbbSHgwHSFZaCtedMoq7z59AQXa6l80UEY+0tAfZvr+JD88c5ncpnvIsCJxz1/Xy1PlebbO7K2eN4MpZI6K1OYIhx84DTew80MzTq8p57M3tPLm8jO9fNYNLZpRErQ4R6R+b9jUQcjA1waam7E4HuftRSsAYV5zLeVMG8+NrTuIfXziLsUU5fPYPy7ln4TragyG/SxSRo7ChogGAKQk49HRXCgIPTRmaz5/vOJ2bTh/Nw29s5/qHllDZ0OJ3WSLSR+v31pOVlsLowmy/S/GUgsBj6akBvnP5dH567UmsKqvlI794g2U7u19nJyKxaENFA5OH5iV0RzEoCKLmilnDeeqzc8lMS+HaBxczb9FWQkc4M0lE/OOcY8PeeqaWJHb/ACgIompqST5Pf/5MLpw2hO/9YwO3/24pTX04tVVEom/b/iZqmtuZMbzA71I8pyCIsvzMNO6//mT+6yPTeHlDJTc8vITa5ja/yxKRbt7cegCAuRMG+VyJ9xQEPjAzbpk7lvuvn83aPfVc++BiKuvViSwSS97aup/hBVmMSvCOYlAQ+Ori6UN59JY57K5p5qMPvMnOA01+lyQiQCjkeGvrAU4fPygh5yjuTkHgs7kTipj/6dNoaOng4/MWs7u62e+SRJLemvI6aprbOX1c4h8WAgVBTDhpZAHzP3UazW1BPvHQYsprD/pdkkhSe2JpGempAc6f2utI+QlFQRAjpg3L57e3nkJtU3v4wjP1GYj4oqLuIE8s281HZg5LmnHCFAQx5MSRBTx6yxz21bdw/UNLONDY6ndJIknnZy9uJhSCf7tgot+lRI2CIMaUjink4ZvmsKu6mRsffpu65na/SxJJGtuqGnliWRnXnzaKkUlwtlAnBUEMOn38IOZ9spQtlY188pElNLQoDES8Fgo57lm4jozUAJ89d4Lf5USVgiBGnTOpmP93/cmsLa/nlkff0RXIIh67/9UtvLKxiq9+cDLFeRl+lxNVCoIYduG0Ifz8ulks31XDnb9fRluHhrEW6W8dwRD3LFzHfS9s4rITh3HTGWP8LinqFAQx7pIZJdx71Uxe37yff39ipQaqE+lHoZDjrvkrePiN7dwydww/vubEpLiArLtoz1ksx+CaOSOpbm7j3mc3kJOewveunJHww+KKeK0jGOLLT6zkubV7+eYlU/n02eP8Lsk3CoI4ccfZ42hs6eCXr2whLSXAdy8/ISn/chHpD+3BEP/2x3d5ZnUFX/ngZD511li/S/KVgiBOmBn/ftEk2oIh5i3aRmqK8Z+XTlMYiByl9mCIL/xxBf9YvZdvfXgqnzorefcEOikI4oiZ8fUPTaE9GOLRf+0gFHJ8+zLtGYj0VXswxN2Pr+DZNQqBrhQEccYsvCeQGjB+/fp2OkKOey6frj4DkSPoGgL/59Jp3HZmch8O6kpBEIfMjG9cMpXUlAC/enUrIef4nyvUgSzSm2DI8aU/r+TZNXv5z0uncatC4H0UBHHKzPjqByeTGjB+8fIWgiHHvVfNVBiIdBMKOb7xl9U8vbKcr39oikKgBwqCOGZmfOnCSQTM+NlLm2ntCPHDq08kPVWXh4hAeAL6e55Zx5+W7ubuD0zgjnPG+11STFIQxDkz44sXTiIjLcAPntvI/sZWHrhhNnmZaX6XJuKrlvYg9z2/kUf/tYPbzhzLFy+c5HdJMUt/OiaIz547gfs+diJLtlVzzYOL2af5DCSJ7a5u5qKfLOKhN7Zzw2mj+NaHp+rsusNQECSQq2eP4JGb57DrQBNX3f8mWyob/C5JJOr21bdw06NvU3ewnd/fdir/fcUMhcARKAgSzNmTivnTHafT2hHio796i3d2VPtdkkjULN9Vw4d+9jrltQf59SdLOXNikd8lxQUFQQKaPnwAT332DAblpnP9Q0t4bk2F3yWJeO65NRVcN28xeZmpPHP3WZwyttDvkuKGgiBBjSzM5sk7z2D6sHw+84flPPLGdpzTyKWSeHbsb+IbT63mzt8vZ2pJPn/5zBmML871u6y4orOGEtjAnHTmf/o07n58Bd9duI53d9fyvatmkJuhj13iX0t7kB88t5FH39xOwIxb547lax+aTEZqit+lxR1fvhHM7IvApwAHrAZucc7pNBcPZKal8MANs/nVa1v50QsbWb2njv/3iZOZNizf79JEjtmzqyv4wfMb2b6/iRtPG81dH5jAkPxMv8uKW1E/NGRmw4G7gVLn3HQgBfh4tOtIJoGA8bnzJvD4p0+jua2DK+7/F/OX7NKhIok7bR3hOQQ+84flpKcE+O2tp3DPFdMVAsfJrz6CVCDLzFKBbKDcpzqSyqnjBvHM3Wdx6thCvvHUau56fAXVTW1+lyXSJ60dQe6av5wFy8q445xx/O2uuZw9qdjvshJC1IPAObcHuA/YBVQAdc65F6JdR7Iqys3gN7ecwlc+OJkX1u7lwh+/xjOrKrR3IDGtvqWdmx95hxfW7eM7l53A1z80lcw09QX0Fz8ODQ0ELgfGAsOAHDO7oYf1bjezpWa2tKqqKtplJrTOQ0VPf/5Mhg/M4nPzl3PH75axp/ag36WJ/C+V9S1c++Bi3tlRzU+uPTEpJ5f3mh+Hhi4Atjvnqpxz7cBfgDO6r+Scm+ecK3XOlRYXa/fPC1OGhk+1+48PTWHR5iou+NFr3P/qFto6Qn6XJgLA1qpGrvrVm+w80MQjN8/hylkj/C4pIfkRBLuA08ws28LXfZ8PrPehDgFSUwLcec54XvzSOZw1sYgfPLeRi37yGk8uK6MjqEAQ/6zYVcPVv3qTg21B/nj7aeoP8JAffQRLgAXAcsKnjgaAedGuQ95vxMBs5n2ylEdvnkNWeir//sRKLvjxayxQIIgPXt9cxSd+vYT8rDSe/MwZzBxR4HdJCc3ioZOwtLTULV261O8ykkYo5Pjn+n387MXNrKuoZ/SgbD533gSunDWctBRdjC7eem5NBXc//i7jB+fy21tPoTgvw++S4paZLXPOlR5xPQWB9MY5x4vrK/nZS5tYs6eeEQOzuPG00VxTOpKBOel+lycJaMGyMr66YCUnjSzg0ZtPYUC25tU4HgoC6TfOOV7eUMm8RdtYsr2ajNQAHzlxGJ88fbR22aXfLNpUxU2Pvs2ZE4p48MbZZKdrKJTjpSAQT2zc28Bv39rBUyv20NwW5MSRBVxTOoIPTS+hUHsJcox2HmjiyvvfZHBeBk99di5Z6bpGoD8oCMRT9S3t/GVZGb9fsostlY2kBIy5E4q4dGYJ508ZzKBcHdeVvtm4t4EbHl5CRzDEAo0c2q8UBBIVzjnWVzTw9KpyFq4qZ3f1Qcxg1sgCPjBlMOdMGswJw/IJBDRDlPxvy3ZWc+tjS8lMC/D7205l4pA8v0tKKAoCiTrnHGv21PPShn28tL6S1XvqACjKTef08UXMGTOQk0cNZMrQPFJ19lHSe3HdPj43fzklAzL53W2nMrIw2++SEo6CQHy3v7GVRZuqeG1TFYu3HWBffSsAWWkpzBwxgFmjBjJrVAGzRhUwOE+jRyaTZ1dXcNfjKzhhWD6P3jxHhxI9oiCQmOKco6zmIMt31bBiVy0rdteyrryO9mD439/wgiymluQzeWguk4fmM3lIHmOKsjXJSAJ6bk0Fn5u/gpNGFvDYLXPIy9Qpol7paxDo/CyJCjNjZGE2Iwuzufyk4UB4hqm15XWHgmHT3gZe2VhJMBQOh4CFp9wcV5TDuOJcxhXnMLYoh/HFuQzOyyA8QonEkzV76vjCH99l5ogBCoEYoiAQ32SmpTB7dCGzR783yXhrR5BtVU1s2tfA1spGtu5vYltVE29tO0BL+3tDXeRmpDK2KBwMEwfnMnFIHhOH5DK6MFv9DzGquqmNO363jEE56fz6k6UKgRiiIJCYkpGawtSSfKaWvH8qzVDIsbe+hW1VTWzb3xj52cSynTX8feV78xqlpwQYV5zDxCF5TIoExKQhuYxSQPiqIxjirvnLqWpsZcGdp1OkPoGYoiCQuBAIGMMKshhWkMWZE4ve91xTawdbqxrZtK+Rzfsa2LSvgRW7ani6a0CkBhhXlMPUknxmjSrQ2UtR9oPnN/Lm1gP88OqZuho9BikIJO7lZKQyc0TB//qCaWrtYEtlI5sr3wuIN7bs56kVewDISA0waUgeU4bmMXloHlNL8pkyNE9nsPSzF9ftY96ibdxw2ig+VjrS73KkBwoCSVg5GamcOLKAE0e+FxDOOfbUHmTFrlpWldWyYW8Dr26q4ollZYfWKc7LYMrQPKYPH8CpYwuZM6aQnAz9VzkW5bUH+fKClUwryedbH57mdznSC50+KkL4moeNextYX1HPhr0NbNhbz4aKBjpCjpSAHQqFU8aEg0GjYh5ZRzDEdb9ezLryep7+/JmM09ARUafTR0WOQlFuBkUTMpg74b3+h+a2DpbuqOHt7dW8vb2ax/61g3mLtmEGIwdmM7Iwi8lD8pkxIp8TRxQwtihHp7R28dMXN/POjhp+eu1JCoEYpyAQ6UV2eipnTyo+NEViS3uQlbtreXt7NZsrG9l5oIn5b++k5V/h01qL8zKYO34Qs0cP5IThA5hWkk9mWnJeELdgWRm/fGUL15SO4IpZw/0uR45AQSDSR5lpKZw6bhCnjht0aFlHMMTWqiaW76ph8bYDvL55P399N3y2UmrAmFKSx8wRBUwanEtRXgbTSvIZMygnoQfhe3VjJV97chVzJwzinium+12O9IGCQOQ4pKYEmBw56+i6U0bhnKO8roXVZXWsLAt3SD+9spyGlo5Dr8nLSGVscQ7jinKYPnwAJwwbwPjBORTnxv/V0m9tPcCdv1/G5CF5PHDDbA0REicUBCL9yMwYXpDF8IIsLp4+FAhfDFfT3Mbe+hbW7qln1Z5adh5oZsn26kN7DxC+WnpkYTbDC7JISzHyM9MYVxweUmPC4FxGFmaTEsN7EgtXlfPlJ1YycmA2v7vtFF05HEd01pCIj/Y3trKuvJ7t+5vYVtXI7pqDlNceJBgJj/2NbYfWzUgNMLYohylD8xgxMJsRA7OYNDSPiYNzff3SDYUcP31xEz9/eQulowfywI2zdeVwjNBZQyJxoCg3430d0t3VNbezpaqRrZWNbNrXwLb9TSzeVk1lQzmhLn/DDS/IYtKQXCYNzWPS4DzGDw6Pu1SQnebp4aam1g6+9Od3eX7tPq4pHcE9V0zX4aA4pCAQiWEDstOYPXogs0cPfN/yUCg8rPemfQ1sjFw1vXFv+MrpzqG9AbLTUxhWkMXowmzGD85lQnEuQwZkMiQ/g8F5mWSlpXCwPUhaipGdnkrA6FNwVDe18fzavfzy5S1U1B3kPy+dxi1zx8R9H0eyUhCIxKFAwBg1KJtRg7K5YNqQQ8vbgyF2VTezpbKR3dXNlNe2sKe2mZ0Hmnl9y37aOkKHedewlICFb2akphhFuRkU52VQlJtOQ0sHZTUH2b6/CYAThuXzk2tP4pSxhUd4V4llCgKRBJKWEmB8cW6PE8B3BEOU17awr6GFyvpW9ta30NYRIistQHvQ0dwWJOgcwVCIYAiCoRBtHSEONLVRWd/Khr0N5GemMWVoHlfPHsHZE4uZPjxfewEJQEEgkiRSUwKH9iJEutIYvCIiSU5BICKS5BQEIiJJTkEgIpLkFAQiIklOQSAikuQUBCIiSU5BICKS5OJi9FEzqwM2Rx4OAOp6uF8E7O+HzXV9z+Ndt6fn+7LscI9juc29PXe0bfa6vb3VdCzr9bXN+oz71uZ4/ox7WhbNf9c91TPaOdfziIZdOedi/gbM68P9pf29reNdt6fn+7LscI9juc29PXe0bfa6vUfT5mP5jI/2M03Gz7i3NsfzZ3yUn6uvn3H3W7wcGnq6D/e92NbxrtvT831ZdrjHsdzm3p472jZ73d6jed9j+Yx7Wq7POHbb3F+fcU/LYvW7633i4tBQX5jZUteHCRgSSbK1OdnaC8nX5mRrL8RGm+Nlj6Av5vldgA+Src3J1l5IvjYnW3shBtqcMHsEIiJybBJpj0BERI6BgkBEJMkpCEREklxSBIGZnWtmr5vZA2Z2rt/1RIOZ5ZjZMjO71O9aosHMpkY+3wVm9hm/64kGM7vCzH5Kz7tGAAAHqUlEQVRtZn8zs4v8rsdrZjbOzB42swV+1+KlyP/d30Q+2+ujsc2YDwIze8TMKs1sTbflF5vZRjPbYmb/cYS3cUAjkAmUeVVrf+in9gJ8DfizN1X2r/5os3NuvXPuTuAaIOZPP+ynNv/VOfdp4GbgWg/LPW791N5tzrnbvK3UG0fZ/quABZHP9rKoFNhfV/F5dQPOBk4G1nRZlgJsBcYB6cBKYBowA1jY7TYYCEReNwT4g99tikJ7LwA+TvgL4lK/2xSNNkdecxnwJvAJv9sUrTZHXvcj4GS/2xTF9i7wuz0et//rwEmRdeZHo76Yn7zeObfIzMZ0W3wKsMU5tw3AzP4IXO6c+z5wuEMhNUCGF3X2l/5or5mdB+QQ/kd10Mz+4ZwLeVr4ceivz9g593fg72b2DDDfu4qPXz99zgbcCzzrnFvubcXHp5//H8edo2k/4aMWI4B3idJRm5gPgl4MB3Z3eVwGnNrbymZ2FfBBoAD4pbeleeKo2uuc+yaAmd0M7I/lEDiMo/2MzyW8S50B/MPTyrxzVG0GPk9472+AmU1wzj3gZXEeONrPeBDwP8AsM/t6JDDiWW/t/znwSzP7MN4NsfI+8RoE1sOyXq+Mc879BfiLd+V47qjae2gF5x7r/1Ki5mg/41eBV70qJkqOts0/J/ylEa+Otr0HgDu9Kyfqemy/c64JuCWahcR8Z3EvyoCRXR6PAMp9qiUakq29oDZD4rc52drbXcy0P16D4B1gopmNNbN0wh2jf/e5Ji8lW3tBbU6GNidbe7uLmfbHfBCY2ePAW8BkMyszs9uccx3AXcDzwHrgz865tX7W2V+Srb2gNidDm5Otvd3Fevs16JyISJKL+T0CERHxloJARCTJKQhERJKcgkBEJMkpCEREkpyCQEQkySkIpN+ZWWMUtnFZH4fj7s9tnmtmZxzD62aZ2UOR+zebWUyMd2VmY7oPi9zDOsVm9ly0ahJ/KAgkZplZSm/POef+7py714NtHm78rXOBow4C4BvAL46pIJ8556qACjOb63ct4h0FgXjKzL5iZu+Y2Soz+06X5X+18Axqa83s9i7LG83su2a2BDjdzHaY2XfMbLmZrTazKZH1Dv1lbWaPmdnPzexNM9tmZldHlgfM7P7INhaa2T86n+tW46tm9j0zew34gpl9xMyWmNkKM3vRzIZEhhC+E/iimb1rZmdF/lp+MtK+d3r6sjSzPGCmc25lD8+NNrOXIr+bl8xsVGT5eDNbHHnP7/a0h2XhWayeMbOVZrbGzK6NLJ8T+T2sNLO3zSwv8pf/65Hf4fKe9mrMLMXMftjls7qjy9N/BaIyU5b4xO8JG3RLvBvQGPl5ETCP8CiLAcITjJwdea4w8jMLWAMMijx2wDVd3msH8PnI/c8CD0Xu3wz8MnL/MeCJyDamER7jHeBqwkNSB4ChhOejuLqHel8F7u/yeCDvXXX/KeBHkfvfBr7cZb35wJmR+6OA9T2893nAk10ed637aeCmyP1bgb9G7i8Erovcv7Pz99ntfT8K/LrL4wGEJzfZBsyJLMsnPMJwNpAZWTYRWBq5P4bIRCnA7cC3IvczgKXA2Mjj4cBqv/9d6ebdLV6HoZb4cFHktiLyOJfwF9Ei4G4zuzKyfGRk+QEgCDzZ7X06hxBfRnjOgZ781YXnXVhnZkMiy84Enogs32tmrxym1j91uT8C+JOZlRD+ct3ey2suAKaZHRpNON/M8pxzDV3WKQGqenn96V3a8zvgB12WXxG5Px+4r4fXrgbuM7P/Cyx0zr1uZjOACufcOwDOuXoI7z0QHt/+JMK/30k9vN9FwMwue0wDCH8m24FKYFgvbZAEoCAQLxnwfefcg+9bGJ5E5gLgdOdcs5m9Sng+aYAW51yw2/u0Rn4G6f3fbGuX+9btZ180dbn/C+DHzrm/R2r9di+vCRBuw8HDvO9B3mvbkfR54C/n3CYzmw1cAnzfzF4gfAinp/f4IrAPODFSc0sP6xjhPa/ne3guk3A7JEGpj0C89Dxwq5nlApjZcDMbTPivzZpICEwBTvNo+28AH430FQwh3NnbFwOAPZH7N3VZ3gDkdXn8AuHRIwGI/MXd3XpgQi/beZPw0MMQPgb/RuT+YsKHfujy/PuY2TCg2Tn3e8J7DCcDG4BhZjYnsk5epPN7AOE9hRBwI+G5crt7HviMmaVFXjspsicB4T2Iw55dJPFNQSCecc69QPjQxltmthpYQPiL9Dkg1cxWAfcQ/uLzwpOEJ/9YAzwILAHq+vC6bwNPmNnrwP4uy58GruzsLAbuBkojnavr6GH2LOfcBsJTSeZ1fy7y+lsiv4cbgS9Elv8b8CUze5vwoaWeap4BvG1m7wLfBP7bOdcGXAv8wsxWAv8k/Nf8/cBNZraY8Jd6Uw/v9xCwDlgeOaX0Qd7b+zoPeKaH10iC0DDUktDMLNc512jh+W7fBuY65/ZGuYYvAg3OuYf6uH42cNA558zs44Q7ji/3tMjD17OI8KTyNX7VIN5SH4EkuoVmVkC40/eeaIdAxK+Ajx3F+rMJd+4aUEv4jCJfmFkx4f4ShUAC0x6BiEiSUx+BiEiSUxCIiCQ5BYGISJJTEIiIJDkFgYhIklMQiIgkuf8PcSJYbK8AxyYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "wd_reg = SetL2WdReg(learner, wd, wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e7152dd0c304f4d94b35c2a005a3d52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=90), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/595 [00:00<?, ?it/s]\n",
      "epoch      trn_loss   val_loss                              \n",
      "    0      6.410485   6.080045  \n",
      "    1      5.854744   5.554591                              \n",
      "    2      5.499891   5.267295                              \n",
      "    3      5.284073   5.086434                              \n",
      "    4      5.124359   4.981118                              \n",
      "    5      5.02161    4.889845                              \n",
      "    6      4.947434   4.83225                               \n",
      "    7      4.885387   4.763245                              \n",
      "    8      4.807454   4.735243                              \n",
      "    9      4.751681   4.680733                              \n",
      "    10     4.719166   4.656826                              \n",
      "    11     4.671483   4.633744                              \n",
      "    12     4.64439    4.595091                              \n",
      "    13     4.62636    4.592117                              \n",
      "    14     4.592964   4.5714                                \n",
      "    15     4.582067   4.557607                              \n",
      "    16     4.559809   4.549836                              \n",
      "    17     4.560221   4.537602                              \n",
      "    18     4.533904   4.528851                              \n",
      "    19     4.512298   4.526451                              \n",
      "    20     4.512502   4.517455                              \n",
      "    21     4.489821   4.504734                              \n",
      "    22     4.474753   4.502141                              \n",
      "    23     4.463279   4.495922                              \n",
      "    24     4.460667   4.486231                              \n",
      "    25     4.454461   4.474008                              \n",
      "    26     4.435255   4.476823                              \n",
      "    27     4.423276   4.465021                              \n",
      "    28     4.421866   4.468977                              \n",
      "    29     4.433668   4.461839                              \n",
      "    30     4.414229   4.450702                              \n",
      "    31     4.435622   4.448021                              \n",
      "    32     4.413757   4.450881                              \n",
      "    33     4.398901   4.446179                              \n",
      "    34     4.387288   4.455989                              \n",
      "    35     4.420079   4.439735                              \n",
      "    36     4.412159   4.442221                              \n",
      "    37     4.382121   4.445404                              \n",
      "    38     4.392087   4.440709                              \n",
      "    39     4.373209   4.436787                              \n",
      "    40     4.367475   4.441681                              \n",
      "    41     4.417702   4.431073                              \n",
      "    42     4.364785   4.426897                              \n",
      "    43     4.352531   4.435394                              \n",
      "    44     4.367915   4.43246                               \n",
      "    45     4.341848   4.417004                              \n",
      "    46     4.335891   4.411017                              \n",
      "    47     4.333953   4.403935                              \n",
      "    48     4.315255   4.407459                              \n",
      "    49     4.325346   4.396338                              \n",
      "    50     4.293844   4.396506                              \n",
      "    51     4.321533   4.37977                               \n",
      "    52     4.272496   4.384592                              \n",
      "    53     4.254086   4.381836                              \n",
      "    54     4.251017   4.377327                              \n",
      "    55     4.249052   4.367684                              \n",
      "    56     4.266515   4.356156                              \n",
      "    57     4.226847   4.359189                              \n",
      "    58     4.201076   4.359402                              \n",
      "    59     4.199045   4.353416                              \n",
      "    60     4.174294   4.346579                              \n",
      "    61     4.188858   4.347184                              \n",
      "    62     4.165584   4.341105                              \n",
      "    63     4.146008   4.33942                               \n",
      "    64     4.167437   4.32025                               \n",
      "    65     4.130958   4.322915                              \n",
      "    66     4.123156   4.319929                              \n",
      "    67     4.126344   4.311067                              \n",
      "    68     4.130268   4.303348                              \n",
      "    69     4.117976   4.30403                               \n",
      "    70     4.068963   4.305081                              \n",
      "    71     4.046324   4.302297                              \n",
      "    72     4.038844   4.297967                              \n",
      "    73     4.046522   4.286183                              \n",
      "    74     4.035809   4.289014                              \n",
      "    75     4.03719    4.283927                              \n",
      "    76     3.991338   4.28167                               \n",
      "    77     3.974408   4.280581                              \n",
      "    78     3.978323   4.275771                              \n",
      "    79     3.95472    4.277403                              \n",
      "    80     3.966804   4.271339                              \n",
      "    81     3.923712   4.270922                              \n",
      "    82     3.962057   4.260138                              \n",
      "    83     3.917894   4.26514                               \n",
      "    84     3.902206   4.264861                              \n",
      "    85     3.913417   4.260015                              \n",
      "    86     3.921006   4.256689                              \n",
      "    87     3.910084   4.256157                              \n",
      "    88     3.926284   4.253355                              \n",
      "    89     3.88931    4.253399                              \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[4.253399471250864]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit_opt_sched(custom_cycle([7.5,37.5,37.5,7.5], 3e-3, opt_fn, 10, 0.8, 0.7, wd), callbacks=[wd_reg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 108/108 [00:31<00:00,  3.46it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor(4.2479, device='cuda:0'), tensor(69.9555))"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_validate(learner, np.concatenate(val_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.Adam, betas=(0.8,0.99))\n",
    "splits = [2800, 20000]\n",
    "learner = get_lm_MOTAS(md, opt_fn, vocab_size, em_sz, nh, 650, nl, 0, splits, 5,\n",
    "    dropouti=drops[0], dropout=drops[1], wdrop=drops[2], dropoute=drops[3], dropouth=drops[4], dropoutl=0.29)\n",
    "learner.metrics = []\n",
    "learner.clip=0.12\n",
    "learner.unfreeze()\n",
    "learner.reg_fn=partial(seq2seq_reg, alpha=2, beta=1)\n",
    "wd = 1.2e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "wd_reg = SetL2WdReg(learner, wd, wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c0bcac7c31c470c813ba9bafd51c28c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=90), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                              \n",
      "    0      6.39829    6.022632  \n",
      " 54%|█████▍    | 324/595 [00:44<00:37,  7.27it/s, loss=6.05]"
     ]
    }
   ],
   "source": [
    "learner.fit_opt_sched(custom_cycle([7.5,37.5,37.5,7.5], 3e-3, opt_fn, 10, 0.8, 0.7, wd), callbacks=[wd_reg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_validate(learner, np.concatenate(val_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.Adam, betas=(0.8,0.99))\n",
    "splits = [2800, 20000]\n",
    "learner = get_lm_MOTAS(md, opt_fn, vocab_size, em_sz, nh, 650, nl, 0, splits, 5,\n",
    "    dropouti=drops[0], dropout=drops[1], wdrop=drops[2], dropoute=drops[3], dropouth=drops[4], dropoutl=0.2)\n",
    "learner.metrics = []\n",
    "learner.clip=0.12\n",
    "learner.unfreeze()\n",
    "learner.reg_fn=partial(seq2seq_reg, alpha=2, beta=1)\n",
    "wd = 1.2e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wd_reg = SetL2WdReg(learner, wd, wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.fit_opt_sched(custom_cycle([7.5,37.5,37.5,7.5], 3e-3, opt_fn, 10, 0.8, 0.7, wd), callbacks=[wd_reg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_validate(learner, np.concatenate(val_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.Adam, betas=(0.8,0.99))\n",
    "splits = [2800, 20000]\n",
    "learner = get_lm_MOTAS(md, opt_fn, vocab_size, em_sz, nh, 650, nl, 0, splits, 15,\n",
    "    dropouti=drops[0], dropout=drops[1], wdrop=drops[2], dropoute=drops[3], dropouth=drops[4], dropoutl=0.29)\n",
    "learner.metrics = []\n",
    "learner.clip=0.12\n",
    "learner.unfreeze()\n",
    "learner.reg_fn=partial(seq2seq_reg, alpha=2, beta=1)\n",
    "wd = 1.2e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wd_reg = SetL2WdReg(learner, wd, wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.fit_opt_sched(custom_cycle([7.5,37.5,37.5,7.5], 3e-3, opt_fn, 10, 0.8, 0.7, wd), callbacks=[wd_reg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_validate(learner, np.concatenate(val_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.Adam, betas=(0.8,0.99))\n",
    "splits = [2800, 20000]\n",
    "learner = get_lm_MOTAS(md, opt_fn, vocab_size, em_sz, nh, 650, nl, 0, splits, 15,\n",
    "    dropouti=drops[0], dropout=drops[1], wdrop=drops[2], dropoute=drops[3], dropouth=drops[4], dropoutl=0.2)\n",
    "learner.metrics = []\n",
    "learner.clip=0.12\n",
    "learner.unfreeze()\n",
    "learner.reg_fn=partial(seq2seq_reg, alpha=2, beta=1)\n",
    "wd = 1.2e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wd_reg = SetL2WdReg(learner, wd, wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.fit_opt_sched(custom_cycle([7.5,37.5,37.5,7.5], 3e-3, opt_fn, 10, 0.8, 0.7, wd), callbacks=[wd_reg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_validate(learner, np.concatenate(val_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creation of the model with the best set of hyper-parameters I've found:\n",
    " - dropouts from the github repo\n",
    " - grad clipping at 0.25\n",
    " - AR and TAR loss with alpha=2 and beta=1\n",
    " - wd=1.2e-6 \n",
    "Changing any one of those didn't yield better results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.SGD, momentum=0.95)\n",
    "learner= md.get_model(opt_fn, em_sz, nh, nl,\n",
    "    dropouti=drops[0], dropout=drops[1], wdrop=drops[2], dropoute=drops[3], dropouth=drops[4])\n",
    "learner.metrics = [accuracy]\n",
    "learner.clip=0.12\n",
    "learner.unfreeze()\n",
    "learner.reg_fn=partial(seq2seq_reg, alpha=2, beta=1)\n",
    "wd = 1.2e-6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LR Finder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7412125ffedf4ed98e13d973c3e7920f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 51/297 [00:09<00:44,  5.55it/s, loss=10.4]\n",
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      28.393529  155.721588 0.053903  \n",
      "\n"
     ]
    }
   ],
   "source": [
    "learner.lr_find(wds=wd, end_lr=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEOCAYAAAB4nTvgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8XHXZ9/HPlb1J23RLtzSlO3SBsgSEshUKiAhUERAEpAJWFEFBH8VbBUUfQQUfBWUpy+1y3yJYEMoiO2WRNYWW7nSD7m26JWnTNMtczx9zUtIwaZI2M2cm832/XvOaM2ebb4Zhrv7OOb/fMXdHRERkbzLCDiAiIslPxUJERFqlYiEiIq1SsRARkVapWIiISKtULEREpFUqFiIi0ioVCxERaVXcioWZPWBmG81sXpN5vczseTNbEjz3bGHbBjObHTxmxCujiIi0TTxbFn8GTm8273rgRXcfCbwYvI5lp7sfGjzOjmNGERFpA4vncB9mNgR40t3HBa8XAxPdfZ2ZDQBmuvuBMbbb7u5d2/Neffr08SFDhux/aBGRNDJr1qxN7l7U2npZiQjTRD93XwcQFIy+LayXZ2ZlQD1wi7s/1tqOhwwZQllZWQdGFRHp/Mzs47asl+hi0VaD3X2tmQ0DXjKzue6+rPlKZjYVmAowePDgRGcUEUkbib4aakNw+IngeWOsldx9bfC8HJgJHNbCetPcvdTdS4uKWm1FiYjIPkp0sZgBXBpMXwo83nwFM+tpZrnBdB/gWGBBwhKKiMinxPPS2QeBN4EDzWy1mV0O3AKcamZLgFOD15hZqZndF2w6GigzsznAy0TPWahYiIiEKG7nLNz9whYWTYqxbhlwRTD9BnBwvHKJiEj7qQe3iIi0KlmvhkqYXfUNvLK4vMXlZhZ7/l722bjJ7md2T+yxbeO+rfl2wRxrtv6e+4i9zu59fmr+njtocbtWsjT/Ow0jM8PIzIjuI9OirzMyjAyDTItOZ5qRYUZGBtHlwevMYL2WPmcRSQ5pXyy219Qz9W+zwo6R9jKMoJgYWRlGTlYGOZkZ0edgOjdrz9fR6czosuyM3c9dc7LompdFQW4W3XJjTxfkZJGZoQIl0lZpXyy6d8nmyauPa9c2e+v07vge6zSu2thTvummn+ynpW1ib9v0/Rvfj5a2bWHfMTO1tE0LfwtAxCHiTkPEibgH0xCJOA3B60gkurzBo/MjHiyLRNdtcMe9cR2nvsGpa4hQWx997GoyXVsfoaYuQuXO+ujrYNmu+gi19Q3UBOu0Rfe8LPp0zaV31xx6F0Sf+3TNpU/XHPp1z2Nw73xKeuZTkJv2/5uIqFhkZ2Ywrrgw7BjSgeoaIuzYVU9VTT07auvZXlNP1a56duyKTm8PllXsrGPT9l1s3l7L8k3befejWrZU137qHwO9C3Io6ZVPSa98RhR1ZfSAbowe0J1BPbvo8JmkjbQvFtL5ZGdm0CM/hx75Oe3etiHibNlRy7qKnazcUs3KLdWs2lLNqi07mb1qK0/MWbt73W55WYwe0J2jhvTimOG9OXxwT7rkZHbknyKSNOI6kGAilZaWusaGknjbsaueReurWLiukkXrK5m7ppJ5aypoiDjZmcZRQ3txxsEDOH1sf3p3zQ07rkirzGyWu5e2up6Khcj+qaqpo+zjrby1bDPPL9jA8k07yMwwjhnWm/OPLOGzY/uRm6UWhyQnFQuRELg7C9dV8fTcdTw+Zw2rtuykd0EOUyYM4fLjh5KfoyO/klxULERCFok4ry3dxF/e+IiXFm2kqFsu3zt1FOeXlpChy3YlSbS1WKgHt0icZGQYJ44q4oEpRzL9ymMY3Cuf6x+dy4X3vsVHm3aEHU+kXVQsRBKgdEgvpl95DLecczAL1lZy+h9e5eGyVWHHEmkzFQuRBDEzLjhqMM9fdyKHD+7JD6Z/wC+fXEAk0jkOBUvnpmIhkmD9C/P42+WfYcqEIdz3+gq+P30OdQ1t63UuEhZdmiESgswM48azxtCrIIffPf8hO3bVc+dFR2i8KklaalmIhMTMuGbSSH565hienb+Bnz8xn85ydaJ0PmpZiITs8uOGsqGyhmmvLmdQzy5MPWF42JFEPkXFQiQJXH/6QazZtpNfPb2IkX27cdJBfcOOJLIHHYYSSQIZGcZt543noP7duO7h2ayvqAk7ksgeVCxEkkRediZ/uuhwqmsbuOnJ+WHHEdmDioVIEhle1JWrTx7B03PX8/LijWHHEdlNxUIkyXz9hGEMKyrgxsfnU1PXEHYcEUDFQiTp5GZl8svJ41i5pZo7X14adhwRQMVCJClNGNGHs8YPZNpry3WyW5KCioVIkvrBZw8kEoHbnlscdhQRFQuRZFXSK59LJxzA9PdWs3BdZdhxJM2pWIgksW+fNJLuednc/O9FYUeRNKdiIZLECvOzufrkEbz6YTmvflgedhxJYyoWIknukmMOoKRXF3719EIadO8LCYmKhUiSy83K5IenH8Si9VX8/Z2VYceRNKViIZICPn/wACYM781vnlnExipdSiuJp2IhkgLMjJsmj2NXfYTvPTxHt2KVhFOxEEkRI/p25cazxvDakk3cqr4XkmC6n4VICvnKUYOZt6aCO2cuIyszg2tPGYmZbsUq8Re3loWZPWBmG81sXpN5vczseTNbEjz3bGHbS4N1lpjZpfHKKJJqzIxffuFgzj1iELe/uIRL7n+HJ+asZUOlzmNIfFm87vlrZicA24G/uvu4YN5vgC3ufouZXQ/0dPcfNtuuF1AGlAIOzAKOcPete3u/0tJSLysri8NfIpJ83J0/v/ERd85cRnnVLgD6dc8FoKRnPuceMYgvH1miVoe0ysxmuXtpq+vF8wbxZjYEeLJJsVgMTHT3dWY2AJjp7gc22+bCYJ1vBK/vCdZ7cG/vpWIh6ai+IcIHayp4f+U25q+tINOM+WsrWbCukrPHD+TW88aTk6VTk9KythaLRJ+z6Ofu6wCCghHrRsPFwKomr1cH80SkmazMDA4f3JPDB39yRDcSce56ZRm/fXYx1bX13H3xEWRlqmDI/knGb1CsdnPM5o+ZTTWzMjMrKy/XUAgiEL2f91UnjeCmyWN5YeFGbn9xSdiRpBNIdLHYEBx+IniOdd/I1UBJk9eDgLWxdubu09y91N1Li4qKOjysSCr76jFDOO+IQdzx8lJmfbwl7DiS4hJdLGYAjVc3XQo8HmOdZ4HTzKxncLXUacE8EWmnn509lv7d8/jpY/M1rpTsl3heOvsg8CZwoJmtNrPLgVuAU81sCXBq8BozKzWz+wDcfQvwC+Dd4HFTME9E2qkgN4sff340C9ZVMn3WqtY3EGlBXK+GSiRdDSUSm7sz+U//YWt1LS99byLZOtktTbT1aih9a0Q6OTPjmpNHsmrLTh6fHfP0n0irVCxE0sCk0X0ZM6A7d768VOcuZJ+oWIikATPjmkkjWL5pB09+oNaFtJ+KhUiaOG1Mfw7s1407X15GZzlXKYmjYiGSJjIyjG+cOIzFG6qYqft5SzupWIikkbPGD2RgYR73vLIs7CiSYlQsRNJIdmYGlx03lLeWb2H2qm1hx5EUomIhkmYuOGow3fKymPaqWhfSdioWImmma24Wlxx9AM/MW89Hm3aEHUdShIqFSBqaMmEIWRkZ3Pf68rCjSIpQsRBJQ32753HO4cX8s2w1m7bvCjuOpAAVC5E0NfWEYdQ1RJj2qloX0joVC5E0NayoK184tJi/vvkRG6tqwo4jSU7FQiSNXT1pJHUNzt0z1bqQvVOxEEljQ/sUcM5hxfzP2x+zoVKtC2mZioVImrv65JFEIs6dLy8NO4okMRULkTQ3uHc+55UO4sF3VrGuYmfYcSRJqViICFedNALH+eNLal1IbCoWIsKgnvl8+cgSHnp3FSs3V4cdR5KQioWIANFzF5kZxu9f+DDsKJKEVCxEBIB+3fOYMmEI/5q9hg83VIUdR5KMioWI7HblicMpyMnitucWhx1FkoyKhYjs1rMgh68fP4xn529gju53IU2oWIjIHi4/fii9CnK4Va0LaULFQkT20DU3i29NHM5rSzbx9vLNYceRJKFiISKfcvHRB9Cnay53qN+FBFQsRORT8rIzmXrCUF5fuon3Vm4NO44kARULEYnpos8cQM/8bO54cUnYUSQJqFiISEwFuVlccfwwXl5cztzVFWHHkZCpWIhIi756zAF0z8vijpfUukh3KhYi0qJuedlMOXYozy3YwKL1lWHHkRCpWIjIXl127BAKcjI1Im2aU7EQkb3qkZ/DVycM4am561i6cXvYcSQkoRQLM/uOmc0zs/lm9t0YyyeaWYWZzQ4eN4SRU0SirjhuKHlZmbqbXhpLeLEws3HA14GjgPHAmWY2Msaqr7n7ocHjpoSGFJE99O6ay8VHD+ax2WtYulEj0qajMFoWo4G33L3a3euBV4AvhpBDRNrhyhOHk5+Txa3P6n4X6SiMYjEPOMHMeptZPnAGUBJjvWPMbI6Z/dvMxiY2oog017trLl8/fhjPzF/P++rVnXYSXizcfSHwa+B54BlgDlDfbLX3gAPcfTxwB/BYrH2Z2VQzKzOzsvLy8jimFhGAK44fSu+CHH79zCLcPew4kkChnOB29/vd/XB3PwHYAixptrzS3bcH008D2WbWJ8Z+prl7qbuXFhUVJSS7SDoryM3i6pNH8NbyLbyxTCPSppOwrobqGzwPBs4BHmy2vL+ZWTB9FNGc+maKJIELPzOYvt1yuWvmsrCjSAKF1c/iETNbADwBXOXuW83sSjO7Mlh+LjDPzOYAtwMXuNq8IkkhNyuTy46LjkirMaPSh3WW3+DS0lIvKysLO4ZIWqisqePYm1/ihAOL+NNXDg87juwHM5vl7qWtrace3CLSbt3zsrno6AP499x1fLRpR9hxJAFULERkn1x27BCyMjK451Wdu0gHKhYisk/6ds/j/CMHMX3WatZs2xl2HIkzFQsR2WffnDgCQGNGpQEVCxHZZ8U9unB+aQkPl61irVoXnZqKhYjsl2+dFLQuZqp10ZmpWIjIfinu0YVzjyjh4XdXs65CrYvOSsVCRPbbVScNJ+KuXt2dmIqFiOy3QT3zOa90EP94Z5VaF51Um4pFcGe77hZ1v5m9Z2anxTuciKSOb00cQcSdu9W66JTa2rK4zN0rgdOAIuBrwC1xSyUiKaekVz7nHjGIB99ZxfqKmrDjSAdra7Gw4PkM4L/dfU6TeSIiAFx10gga3NWruxNqa7GYZWbPES0Wz5pZNyASv1gikopKeuXzxcOK+fvbKymv2hV2HOlAbS0WlwPXA0e6ezWQTfRQlIjIHr41cTh1DRHue3152FGkA7W1WBwDLHb3bWZ2MfATQAPZi8inDCvqypmHDORvb37M1h21YceRDtLWYnEXUG1m44EfAB8Df41bKhFJad8+eQTVtQ088J8VYUeRDtLWYlEf3KluMvAHd/8D0C1+sUQklY3q143PjevPn//zERU768KOIx2grcWiysx+BFwCPGVmmUTPW4iIxHTVSSOo2lXPX9/4KOwo0gHaWiy+DOwi2t9iPVAM/DZuqUQk5Y0rLuSU0X257/UVVNaodZHq2lQsggLxv0ChmZ0J1Li7zlmIyF5995RRVOys4/7XdO4i1bV1uI/zgXeA84DzgbfN7Nx4BhOR1DeuuJDPjevP/a+v0JVRcXLD4/P43sNz4v4+bT0M9WOifSwudfevAkcBP41fLBHpLK49dRQ7auu551X1u4iHOasr2FgV/+FV2losMtx9Y5PXm9uxrYiksVH9ujF5/ED+/MaKhPyopZv1FTsZUJgX9/dp6w/+M2b2rJlNMbMpwFPA0/GLJSKdyXdOGUVdg+530dHqGiJsrNpF/8IucX+vtp7g/j/ANOAQYDwwzd1/GM9gItJ5DO1TwLmHD+J/31qpe3V3oI1Vu3AnqVoWuPsj7n6du1/r7v+KZygR6XyunjQCx7njJd2ru6OsD2401T/sYmFmVWZWGeNRZWaVcU8nIp3GoJ75XHjUYP5ZtoqPN+8IO06nsC64b0joLQt37+bu3WM8url797inE5FO5aqTRpCZYfzhhSVhR+kU1u8uFklyzkJEpCP0657HpROG8NjsNSzdWBV2nJS3rqKG/JxMuudlxf29VCxEJKGuPHE4XbIz+d3zH4YdJeWtr6ihf2EeZvG/camKhYgkVK+CHC4/bihPz13PvDW6Lc7+WJugPhagYiEiIbjihGEUdsnmtucWhx0lpa2vqKF/9/ifrwAVCxEJQfe8bL5x4jBeXlzOrI+3hB0nJdUHHfIG9lDLQkQ6sSkThtCnay63PqtzF/ti0/ZaGiKekD4WEFKxMLPvmNk8M5tvZt+NsdzM7HYzW2pmH5jZ4WHkFJH4yc/J4qqThvPm8s38Z+mmsOOknHVBh7xOe87CzMYBXyc6cu144EwzG9lstc8BI4PHVKL3ABeRTubCowYzoDCP3z67mOidm6WtGvtYdOZzFqOBt9y92t3rgVeALzZbZzLwV496C+hhZgMSHVRE4isvO5NrJo1k9qptvLhwY+sbyG5rE9h7G8IpFvOAE8yst5nlA2cAJc3WKQZWNXm9OpgnIp3MuUcM4oDe+dz63GIiEbUu2mp9xU5yszLokZ+dkPdLeLFw94XAr4HngWeAOUB9s9Vi9TD51LfIzKaaWZmZlZWXl3d4VhGJv+zMDK49ZRSL1lfx9Lx1YcdJGesqahjYo0tCOuRBSCe43f1+dz/c3U8AtgDNB4pZzZ6tjUHA2hj7mebupe5eWlRUFL/AIhJXZ40fyKh+Xfnd8x9S3xAJO05KiPaxSMwhKAjvaqi+wfNg4BzgwWarzAC+GlwVdTRQ4e76J4dIJ5WZYVx36oEsL9/Bv95fE3aclLCuoiZh5ysA4j/6VGyPmFlvoA64yt23mtmVAO5+N9G78J0BLAWqga+FlFNEEuSzY/txcHEhf3hxCZMPLSYnS93AWhKJOBsqaxLWxwJCKhbufnyMeXc3mXbgqoSGEpFQmRnfO20UU/77XR56dyWXHDMk7EhJq3z7LuojntCWhUq3iCSNE0cVceSQntzx0lJq6hrCjpO0Vm+tBmBQr/yEvaeKhYgkDTPj+6cdyMaqXfztzY/DjpO0Vm+N9t4u6ZmYDnmgYiEiSeYzw3pz/Mg+3PXKMrbvan5VvcAnxaK4h1oWIpLGvn/agWzZUcsDr68IO0pSWr21mj5dc+iSk5mw91SxEJGkM76kB6eO6ce9ry5nW3Vt2HGSzuqtOynumbhWBahYiEiS+t5po9heW8+0V5eHHSXprN66M6HnK0DFQkSS1EH9u3PWIQP57/98xMaqmrDjJI1IxFmzdSeD1LIQEYm67tRR1DVE+P0LzUcESl8bq3ZR2xBhkFoWIiJRQ/oUcPHRB/CPd1ayZENV2HGSwsotQR8LFQsRkU9cM2kkBblZ3PzvRWFHSQpLNkaL5vCirgl9XxULEUlqvQpy+PZJI3hp0UbdfhVYsmE7BTmZFPdQy0JEZA+XThjCoJ5d+OVTC2lI8xskfbihihH9upGRkZj7WDRSsRCRpJeXnckPTj+Ihesq034I8w83bGdU38QeggIVCxFJEWcdMoDxJT249dnF7KxNz0EGt+6oZdP2XYzq1y3h761iISIpwcz4yedHs76yhvteS8+Oeh8GV4SN7KeWhYhIi44c0ovTx/bnrleWpWVHvcZioZaFiEgrfvi5g6itT8+OevPXVtIzPzuhNz1qpGIhIillaBp31Ju3toJxxYWYJfZKKFCxEJEU1NhR71dPLww7SsLU1kdYvL6KsQMLQ3l/FQsRSTm9CnK4+uQRvLy4nNeXpEdHvQ83VFHX4Iwr7h7K+6tYiEhK+uox0Y56//fp9OioN39tBQDj1LIQEWm7dOuoN3dNBd1ysxjcK7FDkzdSsRCRlJVOHfXmrKrgkJLChA/z0UjFQkRSVrp01Kupa2DhukrGD+oRWgYVCxFJaenQUW/+2krqI874EhULEZF91tk76s1etQ2Aw1QsRET2XdOOeh92wo56c1ZtY0BhHn27J77ndiMVCxHpFL7TeEe9TthRb87qbaGerwAVCxHpJHp20o56W3bU8vHmag4drGIhItIhOmNHvTmro+cr1LIQEekgTTvqPfre6rDjdIjZK7eRYXDIoHB6bjdSsRCRTuWsQwZwaEkPbn1uMdW19WHH2W9zVm9jZN9uFORmhZpDxUJEOpXGjnobKndx98xlYcfZL+7OnFXbGF8SbqsCVCxEpBMqHdKLyYcO5O5Xl7Nyc3XYcfbZyi3VbK2u49CSnmFHCadYmNm1ZjbfzOaZ2YNmltds+RQzKzez2cHjijByikjq+tHnRpOVYdz05IKwo+yzxs54admyMLNi4Bqg1N3HAZnABTFWfcjdDw0e9yU0pIikvP6FeVwzaSQvLNzAy4s3hh1nn8xetY287AwODOGe282FdRgqC+hiZllAPrA2pBwi0oldduxQhvUp4KYnFrCrPvVGpZ2zahsHFxeSlRn+GYOEJ3D3NcCtwEpgHVDh7s/FWPVLZvaBmU03s5JY+zKzqWZWZmZl5eXlcUwtIqkoJyuDG88ey4pNO7j/9RVhx2mXuoYI89aGO9JsU2EchuoJTAaGAgOBAjO7uNlqTwBD3P0Q4AXgL7H25e7T3L3U3UuLioriGVtEUtSJo4o4dUw//vjSUtZV7Aw7Tput2LSD2voI44rDP18B4RyGOgVY4e7l7l4HPApMaLqCu292913By3uBIxKcUUQ6kRvOHEN9xPnV04vCjtJmyzZuB2BE364hJ4kKo1isBI42s3wzM2ASsMfIX2Y2oMnLs5svFxFpj5Je+XzzxOE8MWctby3fHHacNlkaFIthRQUhJ4kK45zF28B04D1gbpBhmpndZGZnB6tdE1xaO4folVNTEp1TRDqXb04cTnGPLtz4+HzqGiJhx2nVsvLtFPfoQn5OuD23G4Vyit3db3T3g9x9nLtf4u673P0Gd58RLP+Ru4919/HufpK7p07bUUSSUl52JjecNYbFG6q4NwVuwbq0fDvDk+QQFKgHt4ikkc+O7c/pY/vz+xeWsLx8e9hxWhSJOMs27mB4khyCAhULEUkzN00eS15WBtc/OpdIkg5jvq6yhp11DUlzchtULEQkzfTtnsePPz+ad1Zs4cF3V4YdJ6YV5TsAGNZHxUJEJDTnl5YwYXhvbnl6EesrasKO8ylrt0X7gwzq2SXkJJ9QsRCRtGNm3HzOwdRFIvzksXm4J9fhqDXbdmIWHd8qWahYiEhaOqB3AdedOooXFm7gqbnrwo6zhzXbdtKvWx7ZSTAmVKPkSSIikmCXHTuUg4sL+dmM+WzdURt2nN3WbtvJwB7J06oAFQsRSWNZmRn8+kuHsK26jl88lTz3vYgWi+Q5XwEqFiKS5sYM7M43Jw7n0ffW8NQH4R+OikSctRU1FKtYiIgkl2smjWR8SQ+uf/SD0K+O2ryjltr6iFoWIiLJJjszg9svOJTa+gg3zpgXapbGy2ZVLEREktABvQv47imjeHb+Bp6Ztz60HJ8UC53gFhFJSlccP5SD+nfjxhnzqKqpCyXDyi3VAAzqkR/K+7dExUJEJJCdmcEtXzqE8qpdXHzf26zeWp3wDAvXVTKgMI/C/OyEv/feqFiIiDRxaEkP7rzoCJZv2sGlD7xDdW19Qt9//tpKxgzontD3bAsVCxGRZk4f15+7L44WjJueSFz/i5q6BpaVb2fsQBULEZGUcOyIPnzzxOH8491VPPnB2oS85+L1VUQ82vcj2ahYiIi04NpTR3HY4B786NG5rNoS//MX89dWAjB2YGHc36u9VCxERFoQ7X9xGDhc84/3qa2P3727562p4P7Xl1PYJTuphiZvpGIhIrIXJb3yuflLB/P+ym388JEP4jKc+crN1Vx039ts31XP7Rcehpl1+Hvsr6ywA4iIJLszDxnIivId3Pb8h3TPy+JnZ4/tsB/0bdW1TP1bGQD//MYEBvdOrv4VjVQsRETa4Nsnj6Cypo57X1tBRoZxw5lj9rtg1NZHuOT+d1i+aQf3X1qatIUCVCxERNrEzPivM0bTEIEH/rOCrIzo6/0pGK8vLWfumgp+/+VDOX5kUQem7XgqFiIibWRm/PTM0TREItz72gp6FuTwrYkj9nl/z8xbT7fcLM44eEAHpowPneAWEWkHM+PGs8Yy+dCB/OaZxfz97ZUArNpSvdfe3m8s3cS6ip27X9c3RHh+wQYmje5LTlby/xSrZSEi0k4ZGcat542ncmcd//WvuTy3YD0zF5eTnWlce+qoPVob9Q0Rrnt4DjPmrKVbbhY/PXMM5x4xiJcXl7O1uo7Pju0f4l/SdioWIiL7IDszg7suPoJrH5rNv+et54IjS9iyo5bfPLOYVVuquegzBzCuuJAXFm5gxpy1TD1hGO+v3MoPHvmAf7y7kq3VdQztU8Ck0f3C/lPaRMVCRGQf5WVn8qevHM6KzTsYXtSV+oYIN8yYz/RZq3no3VV8++SRvPfxVgYU5vHD0w/CgH+9v4afPTGfqpp6HphSmhKHoAAsHh1MwlBaWuplZWVhxxARobKmjp/PWMAj760G4DuTRnLtqaN2L1+9tZq5qyv4XBKc2DazWe5e2tp6almIiHSw7nnZ3Hb+eI4f2Yd/vLuSiz4zeI/lg3rmM6hn8vapiEXFQkQkTr5wWDFfOKw47BgdIjUOlomISKhULEREpFWhFAszu9bM5pvZPDN70Mzymi3PNbOHzGypmb1tZkPCyCkiIlEJLxZmVgxcA5S6+zggE7ig2WqXA1vdfQTw/4BfJzaliIg0FdZhqCygi5llAflA83sWTgb+EkxPByZZMg7wLiKSJhJeLNx9DXArsBJYB1S4+3PNVisGVgXr1wMVQO9E5hQRkU+EcRiqJ9GWw1BgIFBgZhc3Xy3Gpp/qPWhmU82szMzKysvLOz6siIgA4RyGOgVY4e7l7l4HPApMaLbOaqAEIDhUVQhsab4jd5/m7qXuXlpUlNxjwYuIpLIwOuWtBI42s3xgJzAJaD5OxwzgUuBN4FzgJW9lXJJZs2ZtMrOPiRaWCqAPsKmd2Rq3bc/y5vP29rpxuvnzvmRtLW9Ly9qSr7WYZe1RAAAJLUlEQVTcnf2z3Zesbc3XWm59tq1nbSlfa7k7Ou++fA9i5UpE1paWFwIHtGnv7p7wB/BzYBEwD/gbkAvcBJwdLM8D/gksBd4BhrVj39OC57J9yDWtvcubz9vb6ybZmj+3O2treVta1pZ8bcjdqT/bfcmqzzZxWZPls92X70GsXMn22bb0CGW4D3e/Ebix2ewbmiyvAc7bx90/sa+52rBtrOXN5+3t9RMtPO+rvW3f0rK25GtpOl0+233JGmu+Ptu2ZWlteWvzwvps9+V70PR1Mn4PWtRpRp1tzszKvA0jKSaDVMoKqZU3lbJCauVNpayQWnmTMWtnHu5jWtgB2iGVskJq5U2lrJBaeVMpK6RW3qTL2mlbFiIi0nE6c8tCREQ6iIqFiIi0SsVCRERalZbFwswmmtlrZna3mU0MO09rzKzAzGaZ2ZlhZ2mNmY0OPtfpZvbNsPPsjZl9wczuNbPHzey0sPO0xsyGmdn9ZjY97CyxBN/TvwSf6UVh59mbZP8sm0uG72rKFQsze8DMNprZvGbzTzezxcE9MK5vZTcObCfa+W91kmcF+CHwcHxS7pFrv/O6+0J3vxI4H4jbpX8dlPUxd/86MAX4cryyBrk6Iu9yd788njmba2fuc4DpwWd6diJztjdrGJ9lc+3Mm7Dvaova20sw7AdwAnA4MK/JvExgGTAMyAHmAGOAg4Enmz36AhnBdv2A/03yrKcQvd/HFODMZP9sg23OBt4AvpLsWYPtbgMOT4XPNthuejyz7kfuHwGHBuv8PVEZ9yVrGJ9lB+WN+3e1pUcoPbj3h7u/GuPOeUcBS919OYCZ/QOY7O43A3s7dLOV6FAjcdERWc3sJKCA6P+MO83saXePJGveYD8zgBlm9hTw92TNGtwj5Rbg3+7+XjxydmTeMLQnN9FW+iBgNuHc/qA9WRckNt2ntSevmS0kQd/VlqTcYagW7L7/RWB1MC8mMzvHzO4hOi7VH+Ocrbl2ZXX3H7v7d4n+6N4br0KxF+39bCea2e3B5/t0vMM1066swNVEW27nmtmV8QzWgvZ+tr3N7G7gMDP7UbzD7UVLuR8FvmRmd7H/Q9l0lJhZk+izbK6lzzbs72rqtSxa0Kb7X+xe4P4o0S92GNqVdfcK7n/u+Cht0t7PdiYwM15hWtHerLcDt8cvTqvam3czEMoPRTMxc7v7DuBriQ7TipayJstn2VxLecP+rnaalsXu+18EBvHpW7Umi1TKCqmVN5WyQurlbZRKuVMpKyRx3s5SLN4FRprZUDPLIXpCeEbImVqSSlkhtfKmUlZIvbyNUil3KmWFZM4b1pUA+3EFwYNE791dR7QKXx7MPwP4kOiVBD8OO2eqZU21vKmUNRXzpmLuVMqaink1kKCIiLSqsxyGEhGROFKxEBGRVqlYiIhIq1QsRESkVSoWIiLSKhULERFplYqFhMbMtifgPc5u4zDwHfmeE81swj5sd5iZ3RdMTzGzRI9bFpOZDWk+jHaMdYrM7JlEZZLEU7GQlGdmmS0tc/cZ7n5LHN5zb+OqTQTaXSyA/wLu2KdAIXP3cmCdmR0bdhaJDxULSQpm9n/M7F0z+8DMft5k/mMWvUvgfDOb2mT+djO7yczeBo4xs4/M7Odm9p6ZzTWzg4L1dv8L3cz+HIyI+4aZLTezc4P5GWZ2Z/AeT5rZ043LmmWcaWa/MrNXgO+Y2Vlm9raZvW9mL5hZv2DI6SuBa81stpkdH/yr+5Hg73s31g+qmXUDDnH3OTGWHWBmLwafzYtmNjiYP9zM3gr2eVOslppF7173lJnNMbN5ZvblYP6Rwecwx8zeMbNuQQviteAzfC9W68jMMs3st03+W32jyeLHgKS+Q57sh7C7kOuRvg9ge/B8GjCN6IibGURv9nNCsKxX8NwFmAf0Dl47cH6TfX0EXB1Mfwu4L5ieAvwxmP4z8M/gPcYQvW8AwLlEh1PPAPoTvc/JuTHyzgTubPK6J+weBeEK4LZg+mfA95us93fguGB6MLAwxr5PAh5p8rpp7ieAS4Ppy4DHgukngQuD6SsbP89m+/0S0aHtG18XEr2pznLgyGBed6IjUOcDecG8kUBZMD2E4AY9wFTgJ8F0LlAGDA1eFwNzw/5e6RGfR2cZolxS22nB4/3gdVeiP1avAteY2ReD+SXB/M1AA/BIs/00Djs/i+gtPmN5zKP3BFlgZv2CeccB/wzmrzezl/eS9aEm04OAh8xsANEf4BUtbHMKMMZs9+jT3c2sm7tXNVlnAFDewvbHNPl7/gb8psn8LwTTfwdujbHtXOBWM/s18KS7v2ZmBwPr3P1dAHevhGgrBPijmR1K9PMdFWN/pwGHNGl5FRL9b7IC2AgMbOFvkBSnYiHJwICb3f2ePWaaTST6Q3uMu1eb2Uyi900HqHH3hmb72RU8N9Dyd3tXk2lr9twWO5pM3wH8zt1nBFl/1sI2GUT/hp172e9OPvnbWtPmAd3c/UMzO4Lo4HQ3m9lzRA8XxdrHtcAGYHyQuSbGOka0BfdsjGV5RP8O6YR0zkKSwbPAZWbWFcDMis2sL9F/tW4NCsVBwNFxev/Xid7hLSNobUxs43aFwJpg+tIm86uAbk1ePwd8u/FF8C/35hYCI1p4nzeIDlUN0XMCrwfTbxE9zEST5Xsws4FAtbv/D9GWx+HAImCgmR0ZrNMtOGFfSLTFEQEuIXo/6OaeBb5pZtnBtqOCFglEWyJ7vWpKUpeKhYTO3Z8jehjlTTObC0wn+mP7DJBlZh8AvyD64xgPjxAdInoecA/wNlDRhu1+BvzTzF4DNjWZ/wTwxcYT3MA1QGlwQngBMe7Q5u6LgMLgRHdz1wBfCz6HS4DvBPO/C1xnZu8QPYwVK/PBwDtmNhv4MfBLd68FvgzcYWZzgOeJtgruBC41s7eI/vDviLG/+4jev/q94HLae/ikFXcS8FSMbaQT0BDlIoCZdXX37WbWG3gHONbd1yc4w7VAlbvf18b184Gd7u5mdgHRk92T4xpy73leBSa7+9awMkj86JyFSNSTZtaD6InqXyS6UATuAs5rx/pHED0hbcA2oldKhcLMioiev1Gh6KTUshARkVbpnIWIiLRKxUJERFqlYiEiIq1SsRARkVapWIiISKtULEREpFX/H5zFFLx6s1ZeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.sched.plot(10,25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "RNNs are very robust to large LRs and the gradient clipping will prevent divergence and explosion so we can take a LR near the minimum of the curve (30) (lesser values don't give as good a result). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1c04b2af5944072af2613355c9a8825",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=90), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      6.628112   6.184458   0.162373  \n",
      "    1      6.153111   5.763566   0.19393                    \n",
      "    2      5.871724   5.499626   0.218413                   \n",
      "    3      5.671414   5.311994   0.226338                   \n",
      "    4      5.514653   5.191954   0.2341                     \n",
      "    5      5.382676   5.100694   0.237727                   \n",
      "    6      5.28028    4.994971   0.245732                   \n",
      "    7      5.197885   4.937758   0.248153                   \n",
      "    8      5.129779   4.89894    0.24881                    \n",
      "    9      5.086409   4.888917   0.246337                   \n",
      "    10     5.022318   4.819836   0.254046                   \n",
      "    11     4.961503   4.778642   0.257679                   \n",
      "    12     4.943135   4.797901   0.251169                   \n",
      "    13     4.893374   4.746737   0.257991                   \n",
      "    14     4.858212   4.733589   0.257597                   \n",
      "    15     4.853794   4.757826   0.256396                   \n",
      "    16     4.829774   4.699786   0.260627                   \n",
      "    17     4.816683   4.698032   0.261095                   \n",
      "    18     4.792389   4.696066   0.259225                   \n",
      "    19     4.77915    4.691632   0.2599                     \n",
      "    20     4.770586   4.693496   0.258266                   \n",
      "    21     4.740906   4.656945   0.261809                   \n",
      "    22     4.745472   4.683056   0.261879                   \n",
      "    23     4.736179   4.689861   0.257528                   \n",
      "    24     4.71485    4.642422   0.260784                   \n",
      "    25     4.702522   4.640438   0.264478                   \n",
      "    26     4.711544   4.670192   0.260692                   \n",
      "    27     4.696842   4.630093   0.262287                   \n",
      "    28     4.698566   4.652778   0.259395                   \n",
      "    29     4.697765   4.649539   0.259032                   \n",
      "    30     4.675126   4.613219   0.263693                   \n",
      "    31     4.651553   4.633019   0.262939                   \n",
      "    32     4.656934   4.630734   0.262286                   \n",
      "    33     4.633595   4.67693    0.256976                   \n",
      "    34     4.619785   4.616129   0.262666                   \n",
      "    35     4.618578   4.631188   0.260656                   \n",
      "    36     4.602003   4.591835   0.266242                   \n",
      "    37     4.588308   4.592105   0.267046                   \n",
      "    38     4.574807   4.562205   0.26683                    \n",
      "    39     4.567198   4.577085   0.266835                   \n",
      "    40     4.559919   4.574068   0.266125                   \n",
      "    41     4.551848   4.548969   0.269177                   \n",
      "    42     4.52185    4.538066   0.270832                   \n",
      "    43     4.524078   4.558314   0.266923                   \n",
      "    44     4.511556   4.549604   0.266736                   \n",
      "    45     4.491623   4.539018   0.267617                   \n",
      "    46     4.506206   4.541572   0.268443                   \n",
      "    47     4.453718   4.523647   0.270518                   \n",
      "    48     4.465969   4.528654   0.269667                   \n",
      "    49     4.429361   4.507297   0.272509                   \n",
      "    50     4.420043   4.498888   0.272415                   \n",
      "    51     4.396214   4.490184   0.273711                   \n",
      "    52     4.386297   4.489014   0.273632                   \n",
      "    53     4.368642   4.496087   0.275629                   \n",
      "    54     4.336937   4.473784   0.275443                   \n",
      "    55     4.357731   4.458461   0.276327                   \n",
      "    56     4.285727   4.4505     0.278376                   \n",
      "    57     4.268256   4.435329   0.278349                   \n",
      "    58     4.260642   4.424965   0.279938                   \n",
      "    59     4.236656   4.415108   0.280714                   \n",
      "    60     4.174799   4.396742   0.282579                   \n",
      "    61     4.171991   4.400411   0.282172                   \n",
      "    62     4.147621   4.38713    0.282438                   \n",
      "    63     4.131076   4.378929   0.284217                   \n",
      "    64     4.123507   4.381564   0.283562                   \n",
      "    65     4.125865   4.373787   0.284536                   \n",
      "    66     4.117826   4.373243   0.283837                   \n",
      "    67     4.087021   4.362433   0.284789                   \n",
      "    68     4.063827   4.362609   0.284747                   \n",
      "    69     4.067848   4.355325   0.285965                   \n",
      "    70     4.050684   4.354864   0.28633                    \n",
      "    71     4.045093   4.355526   0.286265                   \n",
      "    72     4.04821    4.346709   0.28592                    \n",
      "    73     4.019781   4.351393   0.286188                   \n",
      "    74     4.003364   4.338651   0.287468                   \n",
      "    75     3.994394   4.339071   0.286403                   \n",
      "    76     4.002044   4.336948   0.28653                    \n",
      "    77     3.986849   4.334219   0.28619                    \n",
      "    78     4.010163   4.330595   0.287425                   \n",
      "    79     3.957584   4.322871   0.288749                   \n",
      "    80     3.946121   4.32205    0.288278                   \n",
      "    81     3.947767   4.313202   0.289981                   \n",
      "    82     3.949915   4.313076   0.2896                     \n",
      "    83     3.960489   4.310744   0.290152                   \n",
      "    84     3.939803   4.310012   0.289535                   \n",
      "    85     3.921758   4.311282   0.289578                   \n",
      "    86     3.972662   4.303909   0.290421                   \n",
      "    87     3.962061   4.303995   0.290575                   \n",
      "    88     3.957885   4.303369   0.290522                   \n",
      "    89     3.909788   4.300513   0.291127                   \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[4.300513040785696, 0.2911274534668408]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit_opt_sched(one_cycle_lin([30,30,30], 30, 10, 0.95, 0.85, wd))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 108/108 [00:39<00:00,  2.75it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor(4.2898, device='cuda:0'), tensor(72.9502))"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_validate(learner.model, np.concatenate(val_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 108/108 [07:50<00:00,  4.36s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor(4.0113, device='cuda:0'), tensor(55.2198))"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_cache_pointer(learner.model, np.concatenate(val_ids), window=3785)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now if we go for a longer cycle..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.SGD, momentum=0.95)\n",
    "learner= md.get_model(opt_fn, em_sz, nh, nl,\n",
    "    dropouti=drops[0], dropout=drops[1], wdrop=drops[2], dropoute=drops[3], dropouth=drops[4])\n",
    "learner.metrics = [accuracy]\n",
    "learner.clip=0.12\n",
    "learner.unfreeze()\n",
    "learner.reg_fn=partial(seq2seq_reg, alpha=2, beta=1)\n",
    "wd = 1.2e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=150), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      6.649408   6.197354   0.167872  \n",
      "    1      6.195533   5.783183   0.200387                   \n",
      "    2      5.918022   5.524694   0.215999                   \n",
      "    3      5.690343   5.356249   0.22505                    \n",
      "    4      5.537789   5.202367   0.233575                   \n",
      "    5      5.414285   5.127078   0.235668                   \n",
      "    6      5.307435   5.041664   0.23517                    \n",
      "    7      5.212184   4.963502   0.246997                   \n",
      "    8      5.149351   4.897415   0.248754                   \n",
      "    9      5.090227   4.867299   0.251758                   \n",
      "    10     5.010004   4.825522   0.254741                   \n",
      "    11     4.962746   4.793763   0.256                      \n",
      "    12     4.947663   4.79204    0.254782                   \n",
      "    13     4.897349   4.757676   0.255848                   \n",
      "    14     4.865117   4.72921    0.256547                   \n",
      "    15     4.832319   4.713204   0.259961                   \n",
      "    16     4.819389   4.694065   0.260804                   \n",
      "    17     4.806942   4.723109   0.258918                   \n",
      "    18     4.762922   4.67524    0.262903                   \n",
      "    19     4.755131   4.658179   0.262187                   \n",
      "    20     4.748625   4.677272   0.261479                   \n",
      "    21     4.730597   4.645309   0.264832                   \n",
      "    22     4.714807   4.630424   0.266005                   \n",
      "    23     4.701996   4.663434   0.261883                   \n",
      "    24     4.678897   4.630925   0.263253                   \n",
      "    25     4.685403   4.634238   0.26028                    \n",
      "    26     4.663449   4.634643   0.26322                    \n",
      "    27     4.659912   4.628077   0.259481                   \n",
      "    28     4.65907    4.634602   0.260313                   \n",
      "    29     4.656164   4.619688   0.263539                   \n",
      "    30     4.646468   4.629173   0.262927                   \n",
      "    31     4.635587   4.617103   0.262959                   \n",
      "    32     4.625095   4.610733   0.264643                   \n",
      "    33     4.609065   4.595483   0.266779                   \n",
      "    34     4.609857   4.605245   0.262682                   \n",
      "    35     4.623473   4.615111   0.263825                   \n",
      "    36     4.612255   4.618142   0.263836                   \n",
      "    37     4.59817    4.590389   0.265178                   \n",
      "    38     4.599087   4.61217    0.26373                    \n",
      "    39     4.580744   4.584697   0.265973                   \n",
      "    40     4.580185   4.580826   0.268722                   \n",
      "    41     4.579307   4.588978   0.265159                   \n",
      "    42     4.617223   4.612496   0.26092                    \n",
      "    43     4.58218    4.59744    0.264395                   \n",
      "    44     4.606566   4.584587   0.265671                   \n",
      "    45     4.577      4.590339   0.262883                   \n",
      "    46     4.603337   4.556019   0.26912                    \n",
      "    47     4.582302   4.576079   0.264861                   \n",
      "    48     4.573652   4.609775   0.2594                     \n",
      "    49     4.570653   4.601252   0.261732                   \n",
      "    50     4.55572    4.594507   0.262381                   \n",
      "    51     4.566146   4.564219   0.26538                    \n",
      "    52     4.541175   4.574361   0.266589                   \n",
      "    53     4.556397   4.563991   0.268445                   \n",
      "    54     4.536065   4.56424    0.266827                   \n",
      "    55     4.554233   4.566941   0.265304                   \n",
      "    56     4.511211   4.567695   0.265307                   \n",
      "    57     4.528295   4.54725    0.267549                   \n",
      "    58     4.514465   4.558794   0.266461                   \n",
      "    59     4.511183   4.567937   0.262473                   \n",
      "    60     4.517674   4.5524     0.266339                   \n",
      "    61     4.492527   4.536125   0.266537                   \n",
      "    62     4.490115   4.563895   0.262317                   \n",
      "    63     4.486087   4.527713   0.269725                   \n",
      "    64     4.501113   4.526193   0.270721                   \n",
      "    65     4.47306    4.561641   0.266379                   \n",
      "    66     4.452989   4.52104    0.27085                    \n",
      "    67     4.457737   4.515522   0.269519                   \n",
      "    68     4.458689   4.524951   0.270057                   \n",
      "    69     4.432127   4.52077    0.269101                   \n",
      "    70     4.429818   4.513038   0.272313                   \n",
      "    71     4.410089   4.514795   0.271919                   \n",
      "    72     4.424086   4.499379   0.27156                    \n",
      "    73     4.410259   4.503758   0.271314                   \n",
      "    74     4.413021   4.512941   0.2687                     \n",
      "    75     4.389966   4.515917   0.268959                   \n",
      "    76     4.387676   4.511017   0.268987                   \n",
      "    77     4.376257   4.502197   0.268254                   \n",
      "    78     4.414433   4.507595   0.272182                   \n",
      "    79     4.368787   4.497667   0.27215                    \n",
      "    80     4.359642   4.480374   0.273635                   \n",
      "    81     4.343029   4.480061   0.274905                   \n",
      "    82     4.328357   4.471262   0.275672                   \n",
      "    83     4.314973   4.462957   0.276393                   \n",
      "    84     4.305716   4.473528   0.274497                   \n",
      "    85     4.308005   4.460176   0.277077                   \n",
      "    86     4.315002   4.456      0.276681                   \n",
      "    87     4.278982   4.445131   0.278608                   \n",
      "    88     4.266938   4.447204   0.278355                   \n",
      "    89     4.250668   4.436924   0.277769                   \n",
      "    90     4.249078   4.438421   0.279021                   \n",
      "    91     4.229262   4.4459     0.27499                    \n",
      "    92     4.248421   4.428647   0.278395                   \n",
      "    93     4.204804   4.424638   0.280523                   \n",
      "    94     4.177278   4.424596   0.278264                   \n",
      "    95     4.155129   4.407901   0.281168                   \n",
      "    96     4.129777   4.407362   0.28111                    \n",
      "    97     4.114565   4.401756   0.281119                   \n",
      "    98     4.100095   4.384579   0.281917                   \n",
      "    99     4.073934   4.37695    0.283603                   \n",
      "   100     4.075444   4.378812   0.283572                   \n",
      "   101     4.082457   4.363163   0.284674                   \n",
      "   102     4.051677   4.364062   0.284058                   \n",
      "   103     4.033063   4.365989   0.283371                   \n",
      "   104     4.010315   4.35532    0.283701                   \n",
      "   105     4.025712   4.359907   0.284852                   \n",
      "   106     4.022362   4.350756   0.285899                   \n",
      "   107     3.981305   4.343925   0.286282                   \n",
      "   108     3.972874   4.345254   0.287117                   \n",
      "   109     4.000131   4.347037   0.286597                   \n",
      "   110     3.960816   4.337091   0.287973                   \n",
      "   111     3.963151   4.342388   0.28625                    \n",
      "   112     3.941941   4.337061   0.286238                   \n",
      "   113     3.957989   4.334799   0.286591                   \n",
      "   114     3.969365   4.326601   0.287773                   \n",
      "   115     3.912359   4.332631   0.287288                   \n",
      "   116     3.940265   4.335148   0.286299                   \n",
      "   117     3.918527   4.330644   0.288163                   \n",
      "   118     3.97689    4.32098    0.287452                   \n",
      "   119     3.888983   4.315356   0.289135                   \n",
      "   120     3.908631   4.315285   0.28926                    \n",
      "   121     3.879578   4.32234    0.288383                   \n",
      "   122     3.881356   4.309876   0.289078                   \n",
      "   123     3.887289   4.310107   0.289748                   \n",
      "   124     3.884033   4.311704   0.289798                   \n",
      "   125     3.893852   4.307212   0.290339                   \n",
      "   126     3.87459    4.303376   0.290613                   \n",
      "   127     3.842517   4.305129   0.290059                   \n",
      "   128     3.855131   4.300945   0.290232                   \n",
      "   129     3.885341   4.297263   0.290891                   \n",
      "   130     3.820358   4.298969   0.290194                   \n",
      "   131     3.81189    4.298539   0.289748                   \n",
      "   132     3.830213   4.299474   0.289739                   \n",
      "   133     3.799521   4.295897   0.291061                   \n",
      "   134     3.824245   4.29446    0.290947                   \n",
      "   135     3.821448   4.287996   0.291904                   \n",
      "   136     3.813346   4.288382   0.291501                   \n",
      "   137     3.793428   4.295241   0.291664                   \n",
      "   138     3.79816    4.290579   0.291963                   \n",
      "   139     3.793121   4.2841     0.292176                   \n",
      "   140     3.842335   4.282172   0.292219                   \n",
      "   141     3.776307   4.276905   0.292594                   \n",
      "   142     3.79465    4.279565   0.292575                   \n",
      "   143     3.787246   4.277176   0.29355                    \n",
      "   144     3.761223   4.275956   0.293182                   \n",
      "   145     3.752886   4.278934   0.2933                     \n",
      "   146     3.758388   4.2772     0.293114                   \n",
      "   147     3.758411   4.276733   0.292665                   \n",
      "   148     3.78082    4.271979   0.29329                    \n",
      "   149     3.783453   4.270148   0.293407                   \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[4.270147887631959, 0.2934068623857171]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit_opt_sched(one_cycle_lin([50,50,50], 30, 10, 0.95, 0.85, wd))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 108/108 [00:39<00:00,  2.72it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor(4.2590, device='cuda:0'), tensor(70.7395))"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_validate(learner.model, np.concatenate(val_ids))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(smerity is at 68 for the first trainign of a model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 108/108 [07:53<00:00,  4.39s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor(3.9722, device='cuda:0'), tensor(53.1025))"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_cache_pointer(learner.model, np.concatenate(val_ids), window=3785)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(smerity is at 52 for the model with cache pointer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}