LM_wikitext.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data can be downloaded from [here](https://einstein.ai/research/the-wikitext-long-term-dependency-language-modeling-dataset). as suggested by smerity, we only add the eos flag at each end of sentence. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "EOS = '<eos>'\n",
    "PATH=Path('../data/wikitext')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_file(filename):\n",
    "    tokens = []\n",
    "    with open(PATH/filename, encoding='utf8') as f:\n",
    "        for line in f:\n",
    "            tokens.append(line.split() + [EOS])\n",
    "    return np.array(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_tok = read_file('wiki.train.tokens')\n",
    "val_tok = read_file('wiki.valid.tokens')\n",
    "tst_tok = read_file('wiki.test.tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(36718, 3760, 4358)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trn_tok), len(val_tok), len(tst_tok)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we numericalize the tokens."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnt = Counter(word for sent in trn_tok for word in sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('the', 113161),\n",
       " (',', 99913),\n",
       " ('.', 73388),\n",
       " ('of', 56889),\n",
       " ('<unk>', 54625),\n",
       " ('and', 50603),\n",
       " ('in', 39453),\n",
       " ('to', 39190),\n",
       " ('<eos>', 36718),\n",
       " ('a', 34237)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnt.most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "itos = [o for o,c in cnt.most_common()]\n",
    "itos.insert(0,'<pad>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33279"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_size = len(itos); vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "stoi = collections.defaultdict(lambda : 5, {w:i for i,w in enumerate(itos)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_ids = np.array([([stoi[w] for w in s]) for s in trn_tok])\n",
    "val_ids = np.array([([stoi[w] for w in s]) for s in val_tok])\n",
    "tst_ids = np.array([([stoi[w] for w in s]) for s in tst_tok])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is the usual AWD LSTM with three layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "em_sz,nh,nl = 400,1150,3\n",
    "bptt, bs = 70, 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_dl = LanguageModelLoader(np.concatenate(trn_ids), bs, bptt)\n",
    "val_dl = LanguageModelLoader(np.concatenate(val_ids), bs, bptt)\n",
    "md = LanguageModelData(PATH, 0, vocab_size, trn_dl, val_dl, bs=bs, bptt=bptt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "drops_jh = np.array([0.25, 0.1, 0.2, 0.02, 0.15]) #Jeremy's dropouts\n",
    "drops_sm = np.array([0.5,0.4,0.5,0.1,0.3]) #Smerity's dropouts from the paper\n",
    "drops = np.array([0.6,0.4,0.5,0.1,0.2]) #Smerity's dropouts from the github repo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training schedule: 1cycle with either a third phase with cosine annealing or linear decay at one hundreth of the lowest lr. The second one seems to be slightly betters, but by a hair."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_cycle(steps,lr,div,max_mom,min_mom, wd):\n",
    "    return [TrainingPhase(steps[0], opt_fn=optim.SGD, lr=(lr/div,lr), lr_decay=DecayType.LINEAR, \n",
    "                          momentum=(max_mom,min_mom), momentum_decay=DecayType.LINEAR, wds=wd),\n",
    "           TrainingPhase(epochs=steps[1], opt_fn=optim.SGD, lr=(lr,lr/div), lr_decay=DecayType.LINEAR, \n",
    "                          momentum=(\n",
    "                              min_mom,max_mom), momentum_decay=DecayType.LINEAR, wds=wd),\n",
    "           TrainingPhase(epochs=steps[2], opt_fn=optim.SGD, lr=lr/div, lr_decay=DecayType.COSINE, \n",
    "                          momentum=max_mom, wds=wd)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_cycle_lin(steps,lr,div,max_mom,min_mom, wd):\n",
    "    return [TrainingPhase(epochs=steps[0], opt_fn=optim.SGD, lr=(lr/div,lr), lr_decay=DecayType.LINEAR, \n",
    "                          momentum=(max_mom,min_mom), momentum_decay=DecayType.LINEAR, wds=wd),\n",
    "           TrainingPhase(epochs=steps[1], opt_fn=optim.SGD, lr=(lr,lr/div), lr_decay=DecayType.LINEAR, \n",
    "                          momentum=(min_mom,max_mom), momentum_decay=DecayType.LINEAR, wds=wd),\n",
    "           TrainingPhase(epochs=steps[2], opt_fn=optim.SGD, lr=(lr/div,lr/(div*100)), lr_decay=DecayType.LINEAR, \n",
    "                          momentum=max_mom, wds=wd)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def custom_cycle(steps, lr, opt_fn, div, max_mom, min_mom, wd):\n",
    "    return [TrainingPhase(steps[0], opt_fn, lr=(lr/div,lr), lr_decay=DecayType.LINEAR, \n",
    "                          momentum=(max_mom,min_mom), momentum_decay=DecayType.LINEAR, wds=wd),\n",
    "            TrainingPhase(steps[1], opt_fn, lr=lr, momentum=min_mom, wds=wd),\n",
    "            TrainingPhase(steps[2], opt_fn, lr=(lr,lr/div), lr_decay=DecayType.LINEAR, \n",
    "                          momentum=(min_mom,max_mom), momentum_decay=DecayType.LINEAR, wds=wd),\n",
    "            TrainingPhase(steps[3], opt_fn, lr=lr/div, lr_decay=DecayType.COSINE, \n",
    "                          momentum=max_mom, wds=wd)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Helper functions for the evaluation of the model at the end. TextReader is rewritten from the LanguageModelLoader class to have a constant bptt and only one batch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextReader():\n",
    "\n",
    "    def __init__(self, nums, bptt, backwards=False):\n",
    "        self.bptt,self.backwards = bptt,backwards\n",
    "        self.data = self.batchify(nums)\n",
    "        self.i,self.iter = 0,0\n",
    "        self.n = len(self.data)\n",
    "\n",
    "    def __iter__(self):\n",
    "        self.i,self.iter = 0,0\n",
    "        while self.i < self.n-1 and self.iter<len(self):\n",
    "            res = self.get_batch(self.i, self.bptt)\n",
    "            self.i += self.bptt\n",
    "            self.iter += 1\n",
    "            yield res\n",
    "\n",
    "    def __len__(self): return self.n // self.bptt \n",
    "\n",
    "    def batchify(self, data):\n",
    "        data = np.array(data)[:,None]\n",
    "        if self.backwards: data=data[::-1]\n",
    "        return T(data)\n",
    "\n",
    "    def get_batch(self, i, seq_len):\n",
    "        source = self.data\n",
    "        seq_len = min(seq_len, len(source) - 1 - i)\n",
    "        return source[i:i+seq_len], source[i+1:i+1+seq_len].view(-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Validation without reinitializing the hidden state."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_validate(model, source, bptt=2000):\n",
    "    data_source = TextReader(source, bptt)\n",
    "    model.eval()\n",
    "    model.reset()\n",
    "    total_loss = 0.\n",
    "    for inputs, targets in tqdm(data_source):\n",
    "        outputs, raws, outs = model(V(inputs))\n",
    "        p_vocab = F.softmax(outputs,1)\n",
    "        for i, pv in enumerate(p_vocab):\n",
    "            targ_pred = pv[targets[i]]\n",
    "            total_loss -= torch.log(targ_pred.detach())\n",
    "    mean = total_loss / (bptt * len(data_source))\n",
    "    return mean, np.exp(mean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cache pointer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot(vec, size=vocab_size):\n",
    "    a = torch.zeros(len(vec), size)\n",
    "    for i,v in enumerate(vec):\n",
    "        a[i,v] = 1.\n",
    "    return V(a)\n",
    "\n",
    "def my_cache_pointer(model, source, scale=1., theta = 0.662, lambd = 0.1279, window=200, bptt=2000):\n",
    "    data_source = TextReader(source, bptt)\n",
    "    model.eval()\n",
    "    model.reset()\n",
    "    total_loss = 0.\n",
    "    targ_history = None\n",
    "    hid_history = None\n",
    "    for inputs, targets in tqdm(data_source):\n",
    "        outputs, raws, outs = model(V(inputs))\n",
    "        p_vocab = F.softmax(outputs * scale,1)\n",
    "        start = 0 if targ_history is None else targ_history.size(0)\n",
    "        targ_history = one_hot(targets) if targ_history is None else torch.cat([targ_history, one_hot(targets)])\n",
    "        hiddens = raws[-1].squeeze() #results of the last layer + remove the batch size.\n",
    "        hid_history = scale * hiddens if hid_history is None else torch.cat([hid_history, scale * hiddens])\n",
    "        for i, pv in enumerate(p_vocab):\n",
    "            #Get the cached values\n",
    "            p = pv\n",
    "            if start + i > 0:\n",
    "                targ_cache = targ_history[:start+i] if start + i <= window else targ_history[start+i-window:start+i]\n",
    "                hid_cache = hid_history[:start+i] if start + i <= window else hid_history[start+i-window:start+i]\n",
    "                all_dot_prods = torch.mv(theta * hid_cache, hiddens[i])\n",
    "                exp_dot_prods = F.softmax(all_dot_prods).unsqueeze(1)\n",
    "                p_cache = (exp_dot_prods.expand_as(targ_cache) * targ_cache).sum(0).squeeze()\n",
    "                p = (1-lambd) * pv + lambd * p_cache\n",
    "            targ_pred = p[targets[i]]\n",
    "            total_loss -= torch.log(targ_pred.detach())\n",
    "        targ_history = targ_history[-window:]\n",
    "        hid_history = hid_history[-window:]\n",
    "    mean = total_loss / (bptt * len(data_source))\n",
    "    return mean, np.exp(mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_validate_scale(model, source, n_scale=100, bptt=200):\n",
    "    data_source = TextReader(source, bptt)\n",
    "    model.eval()\n",
    "    model.reset()\n",
    "    scales = T(np.linspace(0.75,1.25,n_scale)[None, None])\n",
    "    total_loss = torch.zeros(n_scale).cuda()\n",
    "    targ_history = None\n",
    "    hid_history = None\n",
    "    for inputs, targets in tqdm(data_source):\n",
    "        outputs, raws, outs = model(V(inputs))\n",
    "        p_vocab = F.softmax(outputs.unsqueeze(2) * scales,1)\n",
    "        for i, pv in enumerate(p_vocab):\n",
    "            targ_pred = pv[targets[i],:]\n",
    "            total_loss -= torch.log(targ_pred.detach())\n",
    "    means = total_loss / (bptt * len(data_source))\n",
    "    mean = torch.min(means).item()\n",
    "    idx = torch.argmin(means)\n",
    "    return mean, np.exp(mean), scales.squeeze()[idx].item()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creation of the model with the best set of hyper-parameters I've found:\n",
    " - dropouts from the github repo\n",
    " - grad clipping at 0.25\n",
    " - AR and TAR loss with alpha=2 and beta=1\n",
    " - wd=1.2e-6 \n",
    "Changing any one of those didn't yield better results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.SGD, momentum=0.95)\n",
    "learner= md.get_model(opt_fn, em_sz, nh, nl,\n",
    "    dropouti=drops[0], dropout=drops[1], wdrop=drops[2], dropoute=drops[3], dropouth=drops[4])\n",
    "learner.metrics = [accuracy]\n",
    "learner.clip=0.12\n",
    "learner.unfreeze()\n",
    "learner.reg_fn=partial(seq2seq_reg, alpha=2, beta=1)\n",
    "wd = 1.2e-6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LR Finder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5fbf0d7bfea9445f8685118038edab4c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      20.379997  39.807632  0.000257  \n",
      "\n"
     ]
    }
   ],
   "source": [
    "learner.lr_find(wds=wd, end_lr=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEOCAYAAAB4nTvgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VOXd9/HPb7KyhBAgbGFLlFUEweAO4lq1KtZatYvVulBrq7Z9nva2evfpXbtot7utWq1IrdZW27ovUK0bijsgq4DsS0KAACEh+3Y9f8wZGsKESSAzZybzfb9e85pzzpwz881hyC/XWa7LnHOIiIgcSsDvACIiEv9ULEREJCIVCxERiUjFQkREIlKxEBGRiFQsREQkIhULERGJSMVCREQiilqxMLOHzWynma1osayPmb1qZmu955w2tm0ysyXe44VoZRQRkfaJZsviEeC8VstuA153zo0EXvfmw6lxzh3nPS6OYkYREWkHi2Z3H2Y2AnjJOTfem/8UmO6cKzGzQcA859zoMNtVOud6duSz+vXr50aMGHHkoUVEksiiRYt2OedyI62XGoswLQxwzpUAeAWjfxvrZZrZQqARuNs591ykNx4xYgQLFy7sxKgiIl2fmW1uz3qxLhbtNcw5t83MCoA3zGy5c25965XMbCYwE2DYsGGxzigikjRifTXUDu/wE97zznArOee2ec8bgHnApDbWm+WcK3TOFebmRmxFiYjIYYp1sXgBuNqbvhp4vvUKZpZjZhnedD/gVGBlzBKKiMhBonnp7BPA+8BoMysys+uAu4FzzGwtcI43j5kVmtlsb9OxwEIzWwq8SfCchYqFiIiPonbOwjn3xTZeOivMuguB673p94Bjo5VLREQ6Tndwi4hIRPF6NVTM1DU28danpfvnzSz4vH+eFq95z7RY2EGOI7uvZf9nH/iEmR2UObTEWq1LmNdbvk/rbcyCywNmpJhhBgEzAgG8eSNgkBIIrmOtpkPbBcwwb5uW66WYEQgc/j4VkehL+mJRWdvIzMcW+R0j6aUGjLSUAGkpRnpqwJsOzqelBFos8+ZDr6cGSAsEt+mWnkLPjFS6p6fSIyMl+JyeQvcM79lbnpWZRu9uaSpQIh2Q9MWiV7c0Xrr5tAOWhW5qb9kK+M+y0Lzb/1d4Rx3ur6iWn33g/H/mDs55qG28n7GtbbwlzS64fbNzNDcTfHahZ2+6uY3p/dsdvE1Tc3B5k3M0NDXT0OSob2z2pr35pmYaGg+cr6xrDM43Brerb2qmvrGZmvomquobaW5H4y1gkNM9nZwe6fTpkU6f7un06ZlO3x7BR15Od/J6dyMvpxvZ3dLa+S8k0nUlfbFISwkwPi/b7xjSSZxz1DY0U1XfSHVdsHhU1zdSVde0/7m8poGy6np2V9VTVhV8Xl9ayYJN9ZRV1x9UbLIyUsnL6cawPt0ZPTCLUQOyGD0wi/x+PUhL0Wk/SQ5JXyykazEzuqWn0C09BTrUu1hQc7NjT3U9xWU1FO+tobishqKyaor31rC+tJLXV++kyasmaSnG2EG9OGFEH6bk92HKiD706ZHeyT+RSHyIakeCsVRYWOjUN5REW11jExtKq1izYx+rSvaxeEsZi7fupb6xGTOYkJfN9NH9Of/YgYwZ2MvvuCIRmdki51xhxPVULESOTF1jE8uKynlv3W7mrdnJkq17cQ7GDurF5YVDuOz4IWRl6ryHxCcVCxGf7Kqs46Wl23h2cTFLi8rJykjliilDmXl6Af2zMv2OJ3IAFQuROLB0614efncjLy0rIT0lwHWn5fP10wvU0pC4oWIhEkc27qrif19dw4tLtzGwVyZ3ff5Yzhjd1nAuIrHT3mKh6/5EYiC/Xw/u/eIknr3pFLK7pfG1Py/gt6+uobk9N4WIxAEVC5EYmjQsh+e/dSqXHT+E37++luseXcC+2ga/Y4lEpGIhEmOZaSn86rIJ/OSS8by9dheXP/gBuyrr/I4lckgqFiI+MDOuOmk4D18zhQ2llVz3yAKq6xv9jiXSJhULER+dPiqXe784ieXF5dz8+GIam5r9jiQSloqFiM/OPWYgP54xntdX7+Snc1b5HUckLPUNJRIHrjppOBtLq3j43Y1MGdGHz04Y5HckkQOoZSESJ247fwyThvXmv55exobSSr/jiBxAxUIkTqSnBvjDlyaTlmJ88/HF1Dfq/IXEDxULkTgyuHc3fnnZRFaVVHD/vHV+xxHZT8VCJM6cM24AM44bzB/eXMd6HY6SOKFiIRKHfnjhODJSU7hr7mq/o4gAKhYicalfzwxuOuMoXlu1g/fW7fI7joiKhUi8uvbUfPJ6d+Mnc1btH8pVxC8qFiJxKjMthdvOH8OqkgqeXlTkdxxJcioWInHswgmDmDSsN7/696dU1qnvKPGPioVIHDMzfnjhOEr31fHgW+v9jiNJTMVCJM5NHpbDxRMHM+vtDWzbW+N3HElSKhYiCeD7543GAXf/S5fSij9ULEQSwJCc7tw4rYAXlm7jzU93+h1HkpCKhUiC+OaZRzOyf09uf2a5RtaTmFOxEEkQGakp/PaK49hTVc83/rpIHQ1KTKlYiCSQ8XnZ/PoLE1mwqYwfPrcC53SznsRG1IqFmT1sZjvNbEWLZX3M7FUzW+s957Sx7dXeOmvN7OpoZRRJRBdNHMy3zjiafyzcym9fXaOCITERzZbFI8B5rZbdBrzunBsJvO7NH8DM+gA/Ak4ETgB+1FZREUlW3z1nFF84fgj3vLGOX77yqQqGRF3UioVz7m1gT6vFM4BHvelHgUvCbPoZ4FXn3B7nXBnwKgcXHZGkFggYv/j8BL504jAemLeen89dpYIhURXrMbgHOOdKAJxzJWbWP8w6ecDWFvNF3jIRaSEQMH52yXjSUwI8NH8jDU2OH100DjPzO5p0QbEuFu0R7pse9k8mM5sJzAQYNmxYNDOJxCUz40cXjSM1YMx+ZyMNTc389JLxKhjS6WJ9NdQOMxsE4D2Hu7uoCBjaYn4IsC3cmznnZjnnCp1zhbm5uZ0eViQRmBl3fHYsXz+9gL99uIWH393kdyTpgmJdLF4AQlc3XQ08H2adV4BzzSzHO7F9rrdMRNpgZtx23hjOGTeAu+auYsGm1qcLRY5MNC+dfQJ4HxhtZkVmdh1wN3COma0FzvHmMbNCM5sN4JzbA/wEWOA97vSWicghmBm/uXwiQ3K6cdPfPmbnvlq/I0kXYl3lCorCwkK3cOFCv2OI+G719gpm3PcuZ43tz/1fPt7vOBLnzGyRc64w0nq6g1ukixkzsBe3nDWSucu389rKHX7HkS5CxUKkC5o5rYCR/Xvy87mraGxSH1Jy5FQsRLqgtJQA3z9vDBt2VfHPhRq/uyubPX8D976+Nuqfo2Ih0kWdPbY/k4f15g9vrlProgt7ecV23l2/K+qfo2Ih0kWZGTdNP5rivTXMXbHd7zgSJdsrahnYKzPqn6NiIdKFnTmmPwW5PZj19nr1HdUFOefYWVHHgGwVCxE5AoGAccPUAlYUV/D+ht1+x5FOtqeqnvqmZrUsROTIfW5SHv16pvPQ2xv8jiKdbHtF8MZLFQsROWKZaSl89eQRvPlpKWt27PM7jnSiHV6x0GEoEekUXzlpOJlpAWbPV+uiK9leXgeoZSEinaRPj3QuLxzKc4u3sbNCfUZ1FdsrajGD3KyMqH+WioVIkrjutHwampt55L1NfkeRTrKjvJZ+PTNIS4n+r3IVC5EkMbxvD847ZiB//WAzVXWNfseRTrC9opYBvaLfqgAVC5GkcsO0AipqG/nHgq2RV5a4tyNGN+SBioVIUpk8LIfjh+fw8Lsb1QVIFxBsWahYiEgU3DC1gKKyGl75RN2XJ7Lahib2VjeoZSEi0XHOuAEM79udh+ZvUBcgCSyW91iAioVI0kkJGNedls+SrXtZtLnM7zhymHZUxO4eC1CxEElKlx0/hOxuacxSFyAJa39XH2pZiEi0dE9P5aqThvPqqh1s3FXldxw5DDvKvcNQalmISDR99ZThpAUCPPzORr+jyGHYXlFLt7QUemWmxuTzVCxEklT/rExmHDeYJxdtpayq3u840kHbK2oZmJ2JmcXk81QsRJLYDdMKqG1o5q8fbPY7inTQjvLY3b0NKhYiSW3UgCxOH5XLo+9vprahye840gGxGk41RMVCJMndMLWAXZV1PL+k2O8o0k6xHE41RMVCJMmdenRfxgzMYvb8jbpJL0HEcjjVEBULkSRnZsycVsDanZXMW1Pqdxxph1gOpxqiYiEiXDhhMAN6ZWgkvQQR664+QMVCRID01ADXnJLPu+t288m2cr/jSASxHE41RMVCRAD40onD6JGewuz5ukkv3sVyONUQFQsRASC7WxqXTxnKi0u3UVJe43ccOYRYDqcaomIhIvtde2o+zc7xyLub/I4ihxDreyxAxUJEWhjapzvnjx/E4x9toVLjdMetHTEceztExUJEDnD91Hz2aZzuuBbL4VRDVCxE5ACThuUwZUQOD7+jcbrjUayHUw3xpViY2a1mtsLMPjGzb4d5fbqZlZvZEu/x//zIKZKsrp9aQPHeGv61YrvfUaQVP+6xAB+KhZmNB24ATgAmAhea2cgwq853zh3nPe6MaUiRJHf22AHk9+vBbI3THXe2l8f+7m3wp2UxFvjAOVftnGsE3gI+50MOEWlDSsC49rR8lhaVs2CTxumOJ7EeTjXEj2KxAphmZn3NrDtwATA0zHonm9lSM/uXmR0T7o3MbKaZLTSzhaWl6tNGpDNdNnkIOd01Tne82X8Yqqu3LJxzq4BfAK8CLwNLgdbX6H0MDHfOTQTuBZ5r471mOecKnXOFubm5UUwtkny6padw1UnDeX31DjaUVvodRzw7KupiOpxqiC8nuJ1zf3LOTXbOTQP2AGtbvV7hnKv0pucCaWbWz4eoIkntqpNHkJYS4E8apztuxHo41RC/robq7z0PAy4Fnmj1+kDz9oSZnUAw5+5Y5xRJdrlZGXzuuDyeWlTE7so6v+MIsR9ONcSv+yyeNrOVwIvAN51zZWZ2o5nd6L1+GbDCzJYC9wBXOl2SIeKL66fmU9fYzF8/2OJ3FMGfrj4AYnvQy+Ocmxpm2R9bTN8H3BfTUCIS1sgBWZwxOpfHPtjE108vIDMtxe9IScuP4VRDdAe3iEQUHKe7nucWa5xuP/kxnGqIioWIRHTyUX0ZN6gXD83fQHOzjgj7xY/hVENULEQkotA43etLq5i3ZqffcZKWX119gIqFiLTTZycMYlB2Jg+9rcto/eLHcKohKhYi0i5pKQGuOnk472/YzertFX7HSUp+DKcaomIhIu32xSnDyEwLaCQ9n/gxnGqIioWItFtOj3Q+NymPZxcXs6eq3u84SceveyygncXCG3+ilwX9ycw+NrNzox1OROLPNacEb9J74iPdpBdrfgynGtLelsW1zrkK4FwgF/gacHfUUolI3Bo9MItTj+7LY+9vpkEj6cWUH8OphrS3WIR6rLoA+LNzbmmLZSKSZK49NZ/tFbXMWVbid5Sk4ddwqiHtLRaLzOzfBIvFK2aWBehPCpEkdcbo/hzdvycPvq2R9GLFz3ssoP3F4jrgNmCKc64aSCN4KEpEklAgYMycWsCqkgreWbfL7zhJwa/hVEPaWyxOBj51zu01s68A/w2URy+WiMS7GZMG0z8rQyPpxYhfw6mGtLdYPABUm9lE4PvAZuAvUUslInEvIzWFa04dwfy1u/hkm/52jDa/hlMNaW+xaPTGk5gB/N4593sgK3qxRCQRfPnE4fRIT+EhtS6ibnu5P8OphrS3WOwzsx8AVwFzzCyF4HkLEUli2d3SuPKEYby4rITivTV+x+nSSsprGNQ79sOphrS3WFwB1BG832I7kAf8KmqpRCRhfO3UEQA8rHG6o6p4bw15vbv59vntKhZegfgbkG1mFwK1zjmdsxARhuR058IJg/j7R1sor2nwO06XVVxWw5CcOC8WZnY58BHwBeBy4EMzuyyawUQkccycVkBVfRN/+3Cz31G6pNqGJnZX1cd/ywK4g+A9Flc7574KnAD8MHqxRCSRHDM4m9OO7sef391EXWOT33G6nND5oMEJUCwCzrmWw2Pt7sC2IpIEZk4roHRfHc8v3uZ3lC6nuCxYLBKhZfGymb1iZteY2TXAHGBu9GKJSKKZOrIfYwf1YpbG6e50oZZFXryfs3DOfQ+YBUwAJgKznHP/Fc1gIpJYguN057NuZyVvfqpxujvTtr01pATMt64+oAOHkpxzTzvnvuuc+45z7tlohhKRxHThhMEMzs7kQd2k16mKy2oY2CuTVB9GyAs55Ceb2T4zqwjz2GdmGoRXRA6QlhLg2tPy+WjjHhZvKfM7TpdRtLeGwb39a1VAhGLhnMtyzvUK88hyzvWKVUgRSRxXnjCMXpmp/PGt9X5H6TKKy2oYmtPd1wy6oklEOlXPjFSuPmUEr3yyg3U79/kdJ+E1NDVTUu7vDXmgYiEiUXDNKSPITAvw4Fs6d3Gktu2todnBkD5qWYhIF9O3ZwZXFA7luSXFlJSrg8EjUeTdY6HDUCLSJV0/tYBmB3+arw4Gj8TWPdUAOgwlIl3T0D7duWjCIB7/aAt7q+v9jpOwisqC91gM8mmEvBAVCxGJmhunH0V1fRN/eV8dDB6urWXVDO7t7z0WoGIhIlE0ZmAvzhrTn4ff3UhlXaPfcRLS1j3VDOnt7/kKULEQkSi75ayR7K1u4DG1Lg5LUVkNQ/v4e74CfCoWZnarma0ws0/M7NthXjczu8fM1pnZMjOb7EdOETlyE4f2ZvroXB6av4EqtS46pLahiZ376hji85VQ4EOxMLPxwA0Ex8SYCFxoZiNbrXY+MNJ7zAQeiGlIEelUN585kj1V9RocqYNCvc0ma8tiLPCBc67aOdcIvAV8rtU6M4C/uKAPgN5mNijWQUWkcxw/PIepI/sx6+0N1NRrcKT2Cl026/c9FuBPsVgBTDOzvmbWHbgAGNpqnTxga4v5Im+ZiCSoW84aya7Keh7/aIvfURLGVu+GvKQ8DOWcWwX8AngVeBlYCrQ+kGnhNm29wMxmmtlCM1tYWlra6VlFpPNMGdGHkwv68se31lPboNZFexSVVZOeEqB/VobfUfw5we2c+5NzbrJzbhqwB1jbapUiDmxtDAEOGqvROTfLOVfonCvMzc2NXmAR6RS3nj2S0n11/F2ti3Yp2lNDXk43AoFwfz/Hll9XQ/X3nocBlwJPtFrlBeCr3lVRJwHlzrmSGMcUkU52UkFfTsjvwwNqXbTL1rJq37v5CPHrPounzWwl8CLwTedcmZndaGY3eq/PBTYA64CHgJt8yikinezWs0ayo6KOJxcV+R0l7gXvsfD/fAVAqh8f6pybGmbZH1tMO+CbMQ0lIjFxylF9OX54Dg+8uY4rCoeSnqp7g8OpqmtkT1V90rcsRCRJmRm3nDWSbeW1PKXWRZvipWvyEBULEYm5aSP7cdzQ3vzhzXU0NDX7HScu7b/HIk4OQ6lYiEjMmRm3njWS4r01PPtxsd9x4tLWsvgYxyJExUJEfDF9dC7H5mVz35vraFTr4iBFZTV0S0uhb490v6MAKhYi4pPQuYste6p5bslBt1Elva17gpfNmvl/jwWoWIiIj84e259xg3rxB7UuDhJPl82CioWI+CjUuti4q4qXlum+25a2llUzNE7OV4CKhYj47NxxAxgzMIt731hLU/NBXcAlpfLqBvbVNsZFB4IhKhYi4qtAwLj5zJGsL61i7nK1LuA/V0LFwzgWISoWIuK788cPZGT/ntz7xlqa1bpgRXE5APn9evqc5D9ULETEd4GA8a0zj2bNjkrmqHXBS8tKGNG3O6MGqFiIiBzgwgmDGTWgJ799bU1SXxm1q7KO99bv4sIJg+PmsllQsRCROJESML57zig2lFbx7OLkvav71ZU7aHZw4cT4GklaxUJE4sZnjhnI+Lxe/P71tdQ3JmfrYtOuKtJTA4wekOV3lAOoWIhI3DAz/s+5oykqq+EfC5JzNL3Syjpye2bE1SEoULEQkTgzfVQuJ4zow+9eW8u+2ga/48Tc7sp6+vaMj/6gWlKxEJG4Ymbc8dmx7K6q54F56/2OE3O7Kuvo1zPD7xgHUbEQkbgzcWhvLjluMLPf2UiRd4NasggWC7UsRETa5XvnjcGAX73yqd9RYqa52bG7sl4tCxGR9srr3Y3rp+bz/JJtLNm61+84MVFe00Bjs1OxEBHpiG9MP5p+PdP52ZyVONf1uwHZVVkHQL8sFQsRkXbrmZHKd88ZzYJNZby8YrvfcaKuNFQsdM5CRKRjLi8cwqgBPbn75dVd/ka9XZX1AOTqMJSISMekpgS4/YKxbN5dzV/e3+R3nKjatS/UslCxEBHpsOmj+zNtVC73vrGOvdX1fseJml2VdaQGjOxuaX5HOYiKhYgkhDsuGMu+2gZ+//pav6NETem+Ovr2TCcQiK+uPkDFQkQSxOiBWVwxZSiPvb+Zjbuq/I7T6ZxzfLBxN2MH9fI7SlgqFiKSML5zzigyUgPc/a9VfkfpdJ9sq2DrnhrOHz/Q7yhhqViISMLon5XJN6YfxSuf7ODDDbv9jtOpXvlkOwGDs8cO8DtKWCoWIpJQrjutgEHZmfx0zqouM153Q1Mzzy0p5sT8vvSNwyuhQMVCRBJMt/QUvn/eaJYXl/P80q4xot4zHxexdU8N156W73eUNqlYiEjCmTExjwlDsvnVy59S29Dkd5wj0tTsuOf1dUwcks3ZY/v7HadNKhYiknACAeOOC8ayrbyWP72z0e84R+SjjXso3lvD9VML4m50vJZULEQkIZ1Y0JfPHDOA+99cx46KWr/jHLY5y7eRmRbgrDhuVYBPxcLMvmNmn5jZCjN7wswyW71+jZmVmtkS73G9HzlFJL7dfsFYGpodd7600u8oh6WxqZmXV2znrDED6J6e6necQ4p5sTCzPOAWoNA5Nx5IAa4Ms+o/nHPHeY/ZMQ0pIglheN8e3HzG0cxZVsK8T3f6HafDPtlWwa7Kej4Tp/dWtOTXYahUoJuZpQLdgW0+5RCRBDfz9AIKcnvww+dXUFOfWCe7V5VUADBxSLbPSSKLebFwzhUDvwa2ACVAuXPu32FW/byZLTOzp8xsaExDikjCyEhN4WeXHMvWPTXc+0Zi9Ru1evs+eqSnMDSnu99RIvLjMFQOMAPIBwYDPczsK61WexEY4ZybALwGPNrGe800s4VmtrC0tDSasUUkjp18VF8+P3kIs97ewJod+/yO024rSyoYPTArLjsObM2Pw1BnAxudc6XOuQbgGeCUlis453Y75+q82YeA48O9kXNulnOu0DlXmJubG9XQIhLf7vjsWHpmpnLHs8sT4s5u5xyrSyrituPA1vwoFluAk8ysuwUvKj4LOKBXMDMb1GL24tavi4i01qdHOrefP5YFm8p4ctFWv+NEVFJeS0VtI2NULMJzzn0IPAV8DCz3MswyszvN7GJvtVu8S2uXErxy6ppY5xSRxPOFwiGcMKIPd/1rNbsr6yJv4KN/eWOKjx2Y5XOS9jHn4r+51h6FhYVu4cKFfscQEZ+t3bGPC+6Zz0UTB/O/lx/nd5ywnvhoCz94ZjmnHt2XR792Aqkp/t0fbWaLnHOFkdbTHdwi0qWMHJDFzGkFPPNxMe+s3eV3nINs3l3FnS+uZOrIfvz5Gn8LRUckRkoRkQ64+cyR5Pfrwe3PLo+rey8ampr57j+XkhowfnnZBNJTE+dXcOIkFRFpp8y0FO669Fi27Knmt6+t8S3Hup2V3PzEYrbsrqap2fHjFz9h0eYyfn7psQzK7uZbrsMR352RiIgcppMK+vLFE4Yye/4GLpowmGN9uEv6+SXFvLh0G299upMBvTJZu7OSG6bmc9HEwTHPcqTUshCRLuu288fSr2cG//X0MhqammP++Su3VZDXuxtTR+WS0yOdX142gdsvGBvzHJ1BLQsR6bKyu6Vx54zx3PjXRcyev5FvTD8qpp+/sqSCE/P78LsrJ8X0c6NBLQsR6dLOGz+Q844ZyO9eW8PGXVUx+9w9VfWUlNdyzOD47ySwPVQsRKTL+/GMY0hPDfCDZ5YRq3vLVm4L9ig7bnBi3KEdiYqFiHR5A3plcvsFY/lgwx7+uTD6XYFs3VPNEwu2ADAuQbrziETFQkSSwhWFQzkxvw8/nbOKnVEchrWxqZkvz/6QOctKmDIih5we6VH7rFhSsRCRpBAIGHddeix1jc38v+c/6dT3bmxq5o3VO2hsambuiu1s2VPNH740mSdvPCXyxglCV0OJSNIoyO3Jt88eyS9f/pSnFxXx+eOHdMr7vrF6JzMfW8S54wawZsc+CnJ7cH4CDJXaEWpZiEhS+fq0ozgxvw8/fH4F60srO+U9lxbtBeDfK3ewr7aROy8enxADGnWEioWIJJWUgPH7KyeRkRrgW48vprbhyPuOWl4cHMTo2ZtO4a3vn8FpI/t1QtL4omIhIklnYHYmv7l8IqtKKrhr7pGNreacY3nRXibkZTNpWA49M7rm0X0VCxFJSmeOGcD1p+Xz6PubedkbiKijVhSX88h7myirbmC8D31PxVLXLIEiIu3w/fPG8OHGPXz/qaWMz+vFkJzu7d72/fW7ufaRBdR4h7Em5HXtYqGWhYgkrfTUAPd9aRLNDm79+5IOdTb4s7krGdArgxtPP4rC4TmMGZQYw6MeLhULEUlqw/v24OeXHsuizWX89tX2j32xeVc1p4/K5bbzx/DUN04hIzUliin9p2IhIknv4omDuXLKUO6ft57nlxRHXL+8poF9dY3k5STWAEZHQsVCRIRgZ4Mn5Pfhe08uY+nWvYdct7isBoC83u0/x5HoVCxERICM1BQe/Mrx5GZlcNPfPmZvdT3NzS5sL7XFe71ioZaFiEjyyemRzh++PJmd+2r55uMfc/Zv3+LmJxYfVDCKy6oByOutYiEikpSOG9qbOy4Yy7vrdlNUVsNLy0q4f976AwrGtvJaMlID9OvZNXqUbQ/dZyEi0srVp4ygR0YqhSP68MuXV/OrVz5lZUkFv7viONJSAhSX1ZDXuxtmXav/p0NRy0JEpBUz4wuFQ8nv14P7vjSZ731mNHOWlfDD51bQ1Oz5WxyZAAAJm0lEQVQo2lvD4CQ6BAVqWYiIHFJKwPjmGUdTVdfI/fPWs2DTHrbtreXiiYP9jhZTKhYiIu3wvc+MZuygXvzl/U044Myx/f2OFFMqFiIi7WBmXDRxMBclWYsiROcsREQkIhULERGJSMVCREQiUrEQEZGIVCxERCQiFQsREYlIxUJERCJSsRARkYgsXF/ticjMSoHNQDZQDvQDdnXwbULbduT11ssONR+aDress/O29Vp78kXK7de+TbS8HfkuHEnWSHmT9bsQr//PIuWN9b4dDlzrnHvxkO/snOtSD2CW97zwcLftyOutlx1qvkW2cMs6NW9br7UnX6Tcfu3bRMvbke/CkWTVdyGx/p9F87twOPu2Pds557rkYahDV8cj2zbc662XHWr+xUMsOxyH2rat19qTr61pv/dt62Xxnrcj34UjyRpp+2T9LsTr/7PW8535XTicfduuz+wyh6FaM7OFzrlCv3O0VyLlTaSskFh5EykrKG80xVvWrtiyCJnld4AOSqS8iZQVEitvImUF5Y2muMraZVsWIiLSebpyy0JERDqJioWIiESkYiEiIhElZbEws+lmNt/M/mhm0/3OE4mZ9TCzRWZ2od9ZIjGzsd5+fcrMvuF3nkjM7BIze8jMnjezc/3OcyhmVmBmfzKzp/zO0hbvu/qot0+/7HeeQ0mE/dmS39/VhCsWZvawme00sxWtlp9nZp+a2Tozuy3C2zigEsgEiuI8K8B/Af+MTsoDch1xXufcKufcjcDlQFQv++ukvM85524ArgGuiPOsG5xz10UrY1s6mP1S4Clvn14cz1n92p+tcnUkb0y+q23q6B2Cfj+AacBkYEWLZSnAeqAASAeWAuOAY4GXWj36AwFvuwHA3+I869nAlQS/IBfG+771trkYeA/4UiLk9bb7DTA5QbI+Fc39eoTZfwAc563zeCxzdjSrX/uzE/JG9bva1iOVBOOce9vMRrRafAKwzjm3AcDM/g7McM7dBRzq0E0ZkBGNnNA5Wc3sDKAHwf+INWY21znXHK95vfd5AXjBzOYAj0cja2flNTMD7gb+5Zz7OJ6z+qUj2Qm21IcAS/DhyEUHs66MbbqDdSSvma0iBt/VtiTcYag25AFbW8wXecvCMrNLzexB4DHgvihna61DWZ1zdzjnvk3wl+5D0SoUh9DRfTvdzO7x9u/caIcLo0N5gZsJtt4uM7MboxksjI7u275m9kdgkpn9INrhImgr+zPA583sAY68C5POEjZrnO3Pltrat35+VxOvZdEGC7OszbsNnXPPEPxS+6FDWfev4NwjnR+lXTq6b+cB86IVph06mvce4J7oxTmkjmbdDcT8l0QbwmZ3zlUBX4t1mAjayhpP+7OltvL6+V3tMi2LImBoi/khwDafskSSSFlBeaMpkbK2lkjZEykrxGnerlIsFgAjzSzfzNIJnhB+wedMbUmkrKC80ZRIWVtLpOyJlBXiNa9fVwEcwdUDTwAlQAPBCnydt/wCYA3Bqwju8DtnomVVXmXtCtkTKWui5VVHgiIiElFXOQwlIiJRpGIhIiIRqViIiEhEKhYiIhKRioWIiESkYiEiIhGpWIhvzKwyBp9xcTu7ge/Mz5xuZqccxnaTzGy2N32NmcW637KwzGxE6y60w6yTa2YvxyqTxJ6KhSQ8M0tp6zXn3AvOubuj8JmH6ldtOtDhYgHcDtx7WIF85pwrBUrM7FS/s0h0qFhIXDCz75nZAjNbZmY/brH8OQuOEviJmc1ssbzSzO40sw+Bk81sk5n92Mw+NrPlZjbGW2//X+hm9ojXI+57ZrbBzC7zlgfM7H7vM14ys7mh11plnGdmPzezt4BbzewiM/vQzBab2WtmNsDrbvpG4DtmtsTMpnp/dT/t/XwLwv1CNbMsYIJzbmmY14ab2evevnndzIZ5y48ysw+897wzXEvNgiPXzTGzpWa2wsyu8JZP8fbDUjP7yMyyvBbEfG8ffhyudWRmKWb2qxb/Vl9v8fJzQFyPjidHwO9byPVI3gdQ6T2fC8wi2NtmgOBgP9O81/p4z92AFUBfb94Bl7d4r03Azd70TcBsb/oa4D5v+hHgSe8zxhEcMwDgMoLdqQeAgQTHObksTN55wP0t5nNgfy8I1wO/8ab/B/i/LdZ7HDjNmx4GrArz3mcAT7eYb5n7ReBqb/pa4Dlv+iXgi970jaH92ep9P0+wa/vQfDbBAXU2AFO8Zb0I9kDdHcj0lo0EFnrTI/AG5wFmAv/tTWcAC4F8bz4PWO7390qP6Dy6ShflktjO9R6LvfmeBH9ZvQ3cYmaf85YP9ZbvBpqAp1u9T6jb+UUEh/cM5zkXHBNkpZkN8JadBjzpLd9uZm8eIus/WkwPAf5hZoMI/gLe2MY2ZwPjzPb3PN3LzLKcc/tarDMIKG1j+5Nb/DyPAb9ssfwSb/px4Ndhtl0O/NrMfgG85Jybb2bHAiXOuQUAzrkKCLZCgPvM7DiC+3dUmPc7F5jQouWVTfDfZCOwExjcxs8gCU7FQuKBAXc55x48YKHZdIK/aE92zlWb2TyC46YD1Drnmlq9T5333ETb3+26FtPW6rk9qlpM3wv8r3PuBS/r/7SxTYDgz1BziPet4T8/WyTt7tDNObfGzI4n2DHdXWb2b4KHi8K9x3eAHcBEL3NtmHWMYAvulTCvZRL8OaQL0jkLiQevANeaWU8AM8szs/4E/2ot8wrFGOCkKH3+OwRHdwt4rY3p7dwuGyj2pq9usXwfkNVi/t/At0Iz3l/ura0Cjm7jc94j2E01BM8JvONNf0DwMBMtXj+AmQ0Gqp1zfyXY8pgMrAYGm9kUb50s74R9NsEWRzNwFcGxoFt7BfiGmaV5247yWiQQbIkc8qopSVwqFuI759y/CR5Ged/MlgNPEfxl+zKQambLgJ8Q/OUYDU8T7B56BfAg8CFQ3o7t/gd40szmA7taLH8R+FzoBDdwC1DonRBeSZjR2Zxzq4Fs70R3a7cAX/P2w1XArd7ybwPfNbOPCB7GCpf5WOAjM1sC3AH81DlXD1wB3GtmS4FXCbYK7geuNrMPCP7irwrzfrMJjl39sXc57YP8pxV3BjAnzDbSBaiLchHAzHo65yrNrC/wEXCqc257jDN8B9jnnJvdzvW7AzXOOWdmVxI82T0jqiEPnedtYIZzrsyvDBI9OmchEvSSmfUmeKL6J7EuFJ4HgC90YP3jCZ6QNmAvwSulfGFmuQTP36hQdFFqWYiISEQ6ZyEiIhGpWIiISEQqFiIiEpGKhYiIRKRiISIiEalYiIhIRP8fkLVnKEdvBUoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.sched.plot(10,25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "RNNs are very robust to large LRs and the gradient clipping will prevent divergence and explosion so we can take a LR near the minimum of the curve (30) (lesser values don't give as good a result). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1c04b2af5944072af2613355c9a8825",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=90), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      6.628112   6.184458   0.162373  \n",
      "    1      6.153111   5.763566   0.19393                    \n",
      "    2      5.871724   5.499626   0.218413                   \n",
      "    3      5.671414   5.311994   0.226338                   \n",
      "    4      5.514653   5.191954   0.2341                     \n",
      "    5      5.382676   5.100694   0.237727                   \n",
      "    6      5.28028    4.994971   0.245732                   \n",
      "    7      5.197885   4.937758   0.248153                   \n",
      "    8      5.129779   4.89894    0.24881                    \n",
      "    9      5.086409   4.888917   0.246337                   \n",
      "    10     5.022318   4.819836   0.254046                   \n",
      "    11     4.961503   4.778642   0.257679                   \n",
      "    12     4.943135   4.797901   0.251169                   \n",
      "    13     4.893374   4.746737   0.257991                   \n",
      "    14     4.858212   4.733589   0.257597                   \n",
      "    15     4.853794   4.757826   0.256396                   \n",
      "    16     4.829774   4.699786   0.260627                   \n",
      "    17     4.816683   4.698032   0.261095                   \n",
      "    18     4.792389   4.696066   0.259225                   \n",
      "    19     4.77915    4.691632   0.2599                     \n",
      "    20     4.770586   4.693496   0.258266                   \n",
      "    21     4.740906   4.656945   0.261809                   \n",
      "    22     4.745472   4.683056   0.261879                   \n",
      "    23     4.736179   4.689861   0.257528                   \n",
      "    24     4.71485    4.642422   0.260784                   \n",
      "    25     4.702522   4.640438   0.264478                   \n",
      "    26     4.711544   4.670192   0.260692                   \n",
      "    27     4.696842   4.630093   0.262287                   \n",
      "    28     4.698566   4.652778   0.259395                   \n",
      "    29     4.697765   4.649539   0.259032                   \n",
      "    30     4.675126   4.613219   0.263693                   \n",
      "    31     4.651553   4.633019   0.262939                   \n",
      "    32     4.656934   4.630734   0.262286                   \n",
      "    33     4.633595   4.67693    0.256976                   \n",
      "    34     4.619785   4.616129   0.262666                   \n",
      "    35     4.618578   4.631188   0.260656                   \n",
      "    36     4.602003   4.591835   0.266242                   \n",
      "    37     4.588308   4.592105   0.267046                   \n",
      "    38     4.574807   4.562205   0.26683                    \n",
      "    39     4.567198   4.577085   0.266835                   \n",
      "    40     4.559919   4.574068   0.266125                   \n",
      "    41     4.551848   4.548969   0.269177                   \n",
      "    42     4.52185    4.538066   0.270832                   \n",
      "    43     4.524078   4.558314   0.266923                   \n",
      "    44     4.511556   4.549604   0.266736                   \n",
      "    45     4.491623   4.539018   0.267617                   \n",
      "    46     4.506206   4.541572   0.268443                   \n",
      "    47     4.453718   4.523647   0.270518                   \n",
      "    48     4.465969   4.528654   0.269667                   \n",
      "    49     4.429361   4.507297   0.272509                   \n",
      "    50     4.420043   4.498888   0.272415                   \n",
      "    51     4.396214   4.490184   0.273711                   \n",
      "    52     4.386297   4.489014   0.273632                   \n",
      "    53     4.368642   4.496087   0.275629                   \n",
      "    54     4.336937   4.473784   0.275443                   \n",
      "    55     4.357731   4.458461   0.276327                   \n",
      "    56     4.285727   4.4505     0.278376                   \n",
      "    57     4.268256   4.435329   0.278349                   \n",
      "    58     4.260642   4.424965   0.279938                   \n",
      "    59     4.236656   4.415108   0.280714                   \n",
      "    60     4.174799   4.396742   0.282579                   \n",
      "    61     4.171991   4.400411   0.282172                   \n",
      "    62     4.147621   4.38713    0.282438                   \n",
      "    63     4.131076   4.378929   0.284217                   \n",
      "    64     4.123507   4.381564   0.283562                   \n",
      "    65     4.125865   4.373787   0.284536                   \n",
      "    66     4.117826   4.373243   0.283837                   \n",
      "    67     4.087021   4.362433   0.284789                   \n",
      "    68     4.063827   4.362609   0.284747                   \n",
      "    69     4.067848   4.355325   0.285965                   \n",
      "    70     4.050684   4.354864   0.28633                    \n",
      "    71     4.045093   4.355526   0.286265                   \n",
      "    72     4.04821    4.346709   0.28592                    \n",
      "    73     4.019781   4.351393   0.286188                   \n",
      "    74     4.003364   4.338651   0.287468                   \n",
      "    75     3.994394   4.339071   0.286403                   \n",
      "    76     4.002044   4.336948   0.28653                    \n",
      "    77     3.986849   4.334219   0.28619                    \n",
      "    78     4.010163   4.330595   0.287425                   \n",
      "    79     3.957584   4.322871   0.288749                   \n",
      "    80     3.946121   4.32205    0.288278                   \n",
      "    81     3.947767   4.313202   0.289981                   \n",
      "    82     3.949915   4.313076   0.2896                     \n",
      "    83     3.960489   4.310744   0.290152                   \n",
      "    84     3.939803   4.310012   0.289535                   \n",
      "    85     3.921758   4.311282   0.289578                   \n",
      "    86     3.972662   4.303909   0.290421                   \n",
      "    87     3.962061   4.303995   0.290575                   \n",
      "    88     3.957885   4.303369   0.290522                   \n",
      "    89     3.909788   4.300513   0.291127                   \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[4.300513040785696, 0.2911274534668408]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit_opt_sched(one_cycle_lin([30,30,30], 30, 10, 0.95, 0.85, wd))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 108/108 [00:39<00:00,  2.75it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor(4.2898, device='cuda:0'), tensor(72.9502))"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_validate(learner.model, np.concatenate(val_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 108/108 [07:50<00:00,  4.36s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor(4.0113, device='cuda:0'), tensor(55.2198))"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_cache_pointer(learner.model, np.concatenate(val_ids), window=3785)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now if we go for a longer cycle..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.SGD, momentum=0.95)\n",
    "learner= md.get_model(opt_fn, em_sz, nh, nl,\n",
    "    dropouti=drops[0], dropout=drops[1], wdrop=drops[2], dropoute=drops[3], dropouth=drops[4])\n",
    "learner.metrics = [accuracy]\n",
    "learner.clip=0.12\n",
    "learner.unfreeze()\n",
    "learner.reg_fn=partial(seq2seq_reg, alpha=2, beta=1)\n",
    "wd = 1.2e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=150), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      6.649408   6.197354   0.167872  \n",
      "    1      6.195533   5.783183   0.200387                   \n",
      "    2      5.918022   5.524694   0.215999                   \n",
      "    3      5.690343   5.356249   0.22505                    \n",
      "    4      5.537789   5.202367   0.233575                   \n",
      "    5      5.414285   5.127078   0.235668                   \n",
      "    6      5.307435   5.041664   0.23517                    \n",
      "    7      5.212184   4.963502   0.246997                   \n",
      "    8      5.149351   4.897415   0.248754                   \n",
      "    9      5.090227   4.867299   0.251758                   \n",
      "    10     5.010004   4.825522   0.254741                   \n",
      "    11     4.962746   4.793763   0.256                      \n",
      "    12     4.947663   4.79204    0.254782                   \n",
      "    13     4.897349   4.757676   0.255848                   \n",
      "    14     4.865117   4.72921    0.256547                   \n",
      "    15     4.832319   4.713204   0.259961                   \n",
      "    16     4.819389   4.694065   0.260804                   \n",
      "    17     4.806942   4.723109   0.258918                   \n",
      "    18     4.762922   4.67524    0.262903                   \n",
      "    19     4.755131   4.658179   0.262187                   \n",
      "    20     4.748625   4.677272   0.261479                   \n",
      "    21     4.730597   4.645309   0.264832                   \n",
      "    22     4.714807   4.630424   0.266005                   \n",
      "    23     4.701996   4.663434   0.261883                   \n",
      "    24     4.678897   4.630925   0.263253                   \n",
      "    25     4.685403   4.634238   0.26028                    \n",
      "    26     4.663449   4.634643   0.26322                    \n",
      "    27     4.659912   4.628077   0.259481                   \n",
      "    28     4.65907    4.634602   0.260313                   \n",
      "    29     4.656164   4.619688   0.263539                   \n",
      "    30     4.646468   4.629173   0.262927                   \n",
      "    31     4.635587   4.617103   0.262959                   \n",
      "    32     4.625095   4.610733   0.264643                   \n",
      "    33     4.609065   4.595483   0.266779                   \n",
      "    34     4.609857   4.605245   0.262682                   \n",
      "    35     4.623473   4.615111   0.263825                   \n",
      "    36     4.612255   4.618142   0.263836                   \n",
      "    37     4.59817    4.590389   0.265178                   \n",
      "    38     4.599087   4.61217    0.26373                    \n",
      "    39     4.580744   4.584697   0.265973                   \n",
      "    40     4.580185   4.580826   0.268722                   \n",
      "    41     4.579307   4.588978   0.265159                   \n",
      "    42     4.617223   4.612496   0.26092                    \n",
      "    43     4.58218    4.59744    0.264395                   \n",
      "    44     4.606566   4.584587   0.265671                   \n",
      "    45     4.577      4.590339   0.262883                   \n",
      "    46     4.603337   4.556019   0.26912                    \n",
      "    47     4.582302   4.576079   0.264861                   \n",
      "    48     4.573652   4.609775   0.2594                     \n",
      "    49     4.570653   4.601252   0.261732                   \n",
      "    50     4.55572    4.594507   0.262381                   \n",
      "    51     4.566146   4.564219   0.26538                    \n",
      "    52     4.541175   4.574361   0.266589                   \n",
      "    53     4.556397   4.563991   0.268445                   \n",
      "    54     4.536065   4.56424    0.266827                   \n",
      "    55     4.554233   4.566941   0.265304                   \n",
      "    56     4.511211   4.567695   0.265307                   \n",
      "    57     4.528295   4.54725    0.267549                   \n",
      "    58     4.514465   4.558794   0.266461                   \n",
      "    59     4.511183   4.567937   0.262473                   \n",
      "    60     4.517674   4.5524     0.266339                   \n",
      "    61     4.492527   4.536125   0.266537                   \n",
      "    62     4.490115   4.563895   0.262317                   \n",
      "    63     4.486087   4.527713   0.269725                   \n",
      "    64     4.501113   4.526193   0.270721                   \n",
      "    65     4.47306    4.561641   0.266379                   \n",
      "    66     4.452989   4.52104    0.27085                    \n",
      "    67     4.457737   4.515522   0.269519                   \n",
      "    68     4.458689   4.524951   0.270057                   \n",
      "    69     4.432127   4.52077    0.269101                   \n",
      "    70     4.429818   4.513038   0.272313                   \n",
      "    71     4.410089   4.514795   0.271919                   \n",
      "    72     4.424086   4.499379   0.27156                    \n",
      "    73     4.410259   4.503758   0.271314                   \n",
      "    74     4.413021   4.512941   0.2687                     \n",
      "    75     4.389966   4.515917   0.268959                   \n",
      "    76     4.387676   4.511017   0.268987                   \n",
      "    77     4.376257   4.502197   0.268254                   \n",
      "    78     4.414433   4.507595   0.272182                   \n",
      "    79     4.368787   4.497667   0.27215                    \n",
      "    80     4.359642   4.480374   0.273635                   \n",
      "    81     4.343029   4.480061   0.274905                   \n",
      "    82     4.328357   4.471262   0.275672                   \n",
      "    83     4.314973   4.462957   0.276393                   \n",
      "    84     4.305716   4.473528   0.274497                   \n",
      "    85     4.308005   4.460176   0.277077                   \n",
      "    86     4.315002   4.456      0.276681                   \n",
      "    87     4.278982   4.445131   0.278608                   \n",
      "    88     4.266938   4.447204   0.278355                   \n",
      "    89     4.250668   4.436924   0.277769                   \n",
      "    90     4.249078   4.438421   0.279021                   \n",
      "    91     4.229262   4.4459     0.27499                    \n",
      "    92     4.248421   4.428647   0.278395                   \n",
      "    93     4.204804   4.424638   0.280523                   \n",
      "    94     4.177278   4.424596   0.278264                   \n",
      "    95     4.155129   4.407901   0.281168                   \n",
      "    96     4.129777   4.407362   0.28111                    \n",
      "    97     4.114565   4.401756   0.281119                   \n",
      "    98     4.100095   4.384579   0.281917                   \n",
      "    99     4.073934   4.37695    0.283603                   \n",
      "   100     4.075444   4.378812   0.283572                   \n",
      "   101     4.082457   4.363163   0.284674                   \n",
      "   102     4.051677   4.364062   0.284058                   \n",
      "   103     4.033063   4.365989   0.283371                   \n",
      "   104     4.010315   4.35532    0.283701                   \n",
      "   105     4.025712   4.359907   0.284852                   \n",
      "   106     4.022362   4.350756   0.285899                   \n",
      "   107     3.981305   4.343925   0.286282                   \n",
      "   108     3.972874   4.345254   0.287117                   \n",
      "   109     4.000131   4.347037   0.286597                   \n",
      "   110     3.960816   4.337091   0.287973                   \n",
      "   111     3.963151   4.342388   0.28625                    \n",
      "   112     3.941941   4.337061   0.286238                   \n",
      "   113     3.957989   4.334799   0.286591                   \n",
      "   114     3.969365   4.326601   0.287773                   \n",
      "   115     3.912359   4.332631   0.287288                   \n",
      "   116     3.940265   4.335148   0.286299                   \n",
      "   117     3.918527   4.330644   0.288163                   \n",
      "   118     3.97689    4.32098    0.287452                   \n",
      "   119     3.888983   4.315356   0.289135                   \n",
      "   120     3.908631   4.315285   0.28926                    \n",
      "   121     3.879578   4.32234    0.288383                   \n",
      "   122     3.881356   4.309876   0.289078                   \n",
      "   123     3.887289   4.310107   0.289748                   \n",
      "   124     3.884033   4.311704   0.289798                   \n",
      "   125     3.893852   4.307212   0.290339                   \n",
      "   126     3.87459    4.303376   0.290613                   \n",
      "   127     3.842517   4.305129   0.290059                   \n",
      "   128     3.855131   4.300945   0.290232                   \n",
      "   129     3.885341   4.297263   0.290891                   \n",
      "   130     3.820358   4.298969   0.290194                   \n",
      "   131     3.81189    4.298539   0.289748                   \n",
      "   132     3.830213   4.299474   0.289739                   \n",
      "   133     3.799521   4.295897   0.291061                   \n",
      "   134     3.824245   4.29446    0.290947                   \n",
      "   135     3.821448   4.287996   0.291904                   \n",
      "   136     3.813346   4.288382   0.291501                   \n",
      "   137     3.793428   4.295241   0.291664                   \n",
      "   138     3.79816    4.290579   0.291963                   \n",
      "   139     3.793121   4.2841     0.292176                   \n",
      "   140     3.842335   4.282172   0.292219                   \n",
      "   141     3.776307   4.276905   0.292594                   \n",
      "   142     3.79465    4.279565   0.292575                   \n",
      "   143     3.787246   4.277176   0.29355                    \n",
      "   144     3.761223   4.275956   0.293182                   \n",
      "   145     3.752886   4.278934   0.2933                     \n",
      "   146     3.758388   4.2772     0.293114                   \n",
      "   147     3.758411   4.276733   0.292665                   \n",
      "   148     3.78082    4.271979   0.29329                    \n",
      "   149     3.783453   4.270148   0.293407                   \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[4.270147887631959, 0.2934068623857171]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit_opt_sched(one_cycle_lin([50,50,50], 30, 10, 0.95, 0.85, wd))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 108/108 [00:39<00:00,  2.72it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor(4.2590, device='cuda:0'), tensor(70.7395))"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_validate(learner.model, np.concatenate(val_ids))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(smerity is at 68 for the first trainign of a model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 108/108 [07:53<00:00,  4.39s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor(3.9722, device='cuda:0'), tensor(53.1025))"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_cache_pointer(learner.model, np.concatenate(val_ids), window=3785)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(smerity is at 52 for the model with cache pointer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's use Adam now! It's best with a beta2=0.99 and momentums varying from 0.7 to 0.8 on RNNs when using the 1cycle. The test of the parameters are the same."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.Adam, betas=(0.95,0.99))\n",
    "learner= md.get_model(opt_fn, em_sz, nh, nl,\n",
    "    dropouti=drops[0], dropout=drops[1], wdrop=drops[2], dropoute=drops[3], dropouth=drops[4])\n",
    "learner.metrics = [accuracy]\n",
    "learner.clip=0.12\n",
    "learner.unfreeze()\n",
    "learner.reg_fn=partial(seq2seq_reg, alpha=2, beta=1)\n",
    "wd = 1.2e-6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "... with just one exception! Weight decay can be applied in or outside of the loss, and I actually found it's best to apply it at the two places. This is what this callback does."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SetL2WdReg(Callback):\n",
    "    \n",
    "    def __init__(self, learn, l2_reg, wd):\n",
    "        self.learn, self.l2_reg, self.wd = learn, l2_reg, wd\n",
    "    \n",
    "    def on_batch_begin(self):\n",
    "        for group in self.learn.sched.layer_opt.opt_params():\n",
    "            group['wd'] = self.wd\n",
    "            group['weight decay'] = self.l2_reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "wd_reg = SetL2WdReg(learner, wd, wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5464e56327854401bb19eb2ea29cf903",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=90), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      6.54909    6.103155   0.172851  \n",
      "    1      5.956252   5.583831   0.211071                   \n",
      "    2      5.572222   5.266101   0.228548                   \n",
      "    3      5.334347   5.070214   0.239641                   \n",
      "    4      5.162422   4.949874   0.245425                   \n",
      "    5      5.04886    4.856792   0.251655                   \n",
      "    6      4.961344   4.798619   0.256749                   \n",
      "    7      4.906732   4.763905   0.257545                   \n",
      "    8      4.844824   4.715672   0.262278                   \n",
      "    9      4.833605   4.689378   0.262917                   \n",
      "    10     4.764922   4.656389   0.265042                   \n",
      "    11     4.729482   4.633562   0.266447                   \n",
      "    12     4.703      4.613951   0.267779                   \n",
      "    13     4.675798   4.601596   0.268685                   \n",
      "    14     4.648645   4.581091   0.270207                   \n",
      "    15     4.638086   4.581454   0.270889                   \n",
      "    16     4.6159     4.55567    0.271272                   \n",
      "    17     4.594086   4.544444   0.271662                   \n",
      "    18     4.576698   4.539344   0.272674                   \n",
      "    19     4.573753   4.529755   0.273705                   \n",
      "    20     4.545755   4.525056   0.273994                   \n",
      "    21     4.522114   4.513895   0.275362                   \n",
      "    22     4.53863    4.513391   0.275059                   \n",
      "    23     4.509168   4.501223   0.274745                   \n",
      "    24     4.499395   4.502591   0.275373                   \n",
      "    25     4.52087    4.501964   0.27498                    \n",
      "    26     4.484329   4.489485   0.275954                   \n",
      "    27     4.465084   4.479829   0.277936                   \n",
      "    28     4.474857   4.487589   0.276617                   \n",
      "    29     4.452875   4.476998   0.276644                   \n",
      "    30     4.441264   4.474218   0.277031                   \n",
      "    31     4.450381   4.464353   0.278148                   \n",
      "    32     4.444392   4.469567   0.277042                   \n",
      "    33     4.449705   4.469483   0.277039                   \n",
      "    34     4.425951   4.464804   0.278024                   \n",
      "    35     4.404301   4.457297   0.27761                    \n",
      "    36     4.409717   4.461115   0.277379                   \n",
      "    37     4.442051   4.460232   0.278992                   \n",
      "    38     4.432344   4.44968    0.279477                   \n",
      "    39     4.402722   4.450478   0.278478                   \n",
      "    40     4.422627   4.445634   0.279499                   \n",
      "    41     4.407962   4.447051   0.279828                   \n",
      "    42     4.406973   4.447298   0.279845                   \n",
      "    43     4.388199   4.440246   0.280356                   \n",
      "    44     4.390545   4.429831   0.280839                   \n",
      "    45     4.381222   4.434474   0.280432                   \n",
      "    46     4.382427   4.427907   0.280844                   \n",
      "    47     4.35082    4.41595    0.28236                    \n",
      "    48     4.36585    4.417766   0.282206                   \n",
      "    49     4.334571   4.413389   0.282859                   \n",
      "    50     4.327747   4.409997   0.282827                   \n",
      "    51     4.303554   4.395702   0.284378                   \n",
      "    52     4.320728   4.399799   0.284345                   \n",
      "    53     4.287744   4.38931    0.284061                   \n",
      "    54     4.260646   4.387244   0.284449                   \n",
      "    55     4.270296   4.38372    0.284403                   \n",
      "    56     4.250675   4.382971   0.285356                   \n",
      "    57     4.230201   4.37591    0.285541                   \n",
      "    58     4.247357   4.367116   0.286442                   \n",
      "    59     4.21622    4.357192   0.286695                   \n",
      "    60     4.206273   4.355672   0.28728                    \n",
      "    61     4.197065   4.350989   0.28707                    \n",
      "    62     4.167345   4.347983   0.28748                    \n",
      "    63     4.174051   4.341975   0.288212                   \n",
      "    64     4.14944    4.34131    0.288218                   \n",
      "    65     4.126779   4.342774   0.289614                   \n",
      "    66     4.123692   4.33127    0.289402                   \n",
      "    67     4.098388   4.327804   0.289872                   \n",
      "    68     4.102822   4.324684   0.290459                   \n",
      "    69     4.068126   4.315383   0.290267                   \n",
      "    70     4.056528   4.31608    0.290821                   \n",
      "    71     4.072526   4.310127   0.291618                   \n",
      "    72     4.054048   4.302786   0.292085                   \n",
      "    73     4.0296     4.301126   0.292128                   \n",
      "    74     4.008604   4.296179   0.292674                   \n",
      "    75     3.97632    4.294218   0.293035                   \n",
      "    76     3.991777   4.288872   0.292811                   \n",
      "    77     3.980112   4.286055   0.293719                   \n",
      "    78     3.981714   4.285016   0.294233                   \n",
      "    79     3.945956   4.279977   0.294916                   \n",
      "    80     3.964455   4.274727   0.295062                   \n",
      "    81     3.912232   4.274124   0.294543                   \n",
      "    82     3.890074   4.274893   0.295449                   \n",
      "    83     3.928548   4.268954   0.295029                   \n",
      "    84     3.890957   4.268934   0.295232                   \n",
      "    85     3.931715   4.263895   0.295816                   \n",
      "    86     3.90646    4.263581   0.295572                   \n",
      "    87     3.889267   4.260135   0.296017                   \n",
      "    88     3.876326   4.258806   0.296032                   \n",
      "    89     3.894654   4.259729   0.296142                   \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[4.259729146148415, 0.296141539767594]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit_opt_sched(custom_cycle([7.5,37.5,37.5,7.5], 5e-3, opt_fn, 10, 0.8, 0.7, wd), callbacks=[wd_reg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 108/108 [00:39<00:00,  2.74it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor(4.2518, device='cuda:0'), tensor(70.2312))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_validate(learner.model, np.concatenate(val_ids))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In just 90 epochs, we beat the result of SGD in 150 epochs!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 108/108 [07:50<00:00,  4.36s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor(3.9728, device='cuda:0'), tensor(53.1309))"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_cache_pointer(learner.model, np.concatenate(val_ids), window=3785)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Little trick learned from Jeremy: by rescaling the pre-softmax activations we can get a better loss (and then ppl) without additional training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1088/1088 [01:28<00:00, 12.35it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(4.247614860534668, 69.93840041351893, 0.9772727489471436)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_validate_scale(learner.model, np.concatenate(val_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 108/108 [07:49<00:00,  4.35s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor(3.9751, device='cuda:0'), tensor(53.2563))"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_cache_pointer(learner.model, np.concatenate(val_ids), scale=0.97727275, window=3785)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It doeesn't transfer with the cache pointer, my guess being that we need to retune its hyper-parameter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}