Initialize the bias in the final layer of an SSD.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.conv_learner import *\n",
    "from fastai.dataset import *\n",
    "\n",
    "from pathlib import Path\n",
    "import json\n",
    "from PIL import ImageDraw, ImageFont\n",
    "from matplotlib import patches, patheffects\n",
    "from ipywidgets import FloatProgress\n",
    "from IPython.display import display"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path('../data/pascal')\n",
    "JPEGS = 'VOCdevkit/VOC2007/JPEGImages'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_json = json.load((PATH/'pascal_train2007.json').open())\n",
    "cats = {o['id']:o['name'] for o in trn_json['categories']}\n",
    "imgs_fn = {o['id']:o['file_name'] for o in trn_json['images']}\n",
    "imgs_id = [o['id'] for o in trn_json['images']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2501"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn_anno = collections.defaultdict(lambda:[])\n",
    "for annot in trn_json['annotations']:\n",
    "    if annot['ignore'] == 0:\n",
    "        bb = annot['bbox']\n",
    "        #Transforms bb which is left,top, width, height into top, left, bottom, right.\n",
    "        bb = [bb[1],bb[0],bb[1] + bb[3], bb[0]+bb[2]]\n",
    "        trn_anno[annot[('image_id')]].append((bb, annot['category_id']))\n",
    "len(trn_anno)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "CLAS_CSV = PATH/'tmp/clas.csv'\n",
    "MBB_CSV = PATH/'tmp/mbb.csv'\n",
    "\n",
    "f_model=resnet34\n",
    "size=224\n",
    "batch_size=32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([array([6]), array([14, 12]), array([ 1,  1, 14, 14, 14]), ..., array([17,  8, 14, 14, 14]),\n",
       "       array([6]), array([11])], dtype=object)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annot_cats = [[cats[int(ann[1])] for ann in trn_anno[i]] for i in imgs_id]\n",
    "id2cats = list(cats.values())\n",
    "cats2id = {c:i for i,c in enumerate(id2cats)}\n",
    "model_cats = np.array([np.array([cats2id[c] for c in ac]) for ac in annot_cats])\n",
    "model_cats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_idx = get_cv_idxs(len(imgs_id))\n",
    "((val_cats, trn_cats),) = split_by_idx(val_idx, model_cats)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Bboxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_bbs = [np.concatenate([ann[0] for ann in trn_anno[i]]) for i in imgs_id]\n",
    "model_bbsc = [' '.join([str(p) for p in o]) for o in model_bbs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['96 155 270 351',\n",
       " '61 184 199 279 77 89 336 403',\n",
       " '229 8 500 245 219 229 500 334 0 1 369 117 1 2 462 243 0 224 486 334',\n",
       " '124 89 212 337',\n",
       " '77 103 183 375 87 132 123 197 179 194 229 213 188 25 238 44']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_bbsc[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[([229, 8, 500, 245], 2),\n",
       " ([219, 229, 500, 334], 2),\n",
       " ([0, 1, 369, 117], 15),\n",
       " ([1, 2, 462, 243], 15),\n",
       " ([0, 224, 486, 334], 15)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn_anno[imgs_id[2]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'fn': [imgs_fn[i] for i in imgs_id], 'bbs': model_bbsc},columns=['fn','bbs'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fn</th>\n",
       "      <th>bbs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000012.jpg</td>\n",
       "      <td>96 155 270 351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000017.jpg</td>\n",
       "      <td>61 184 199 279 77 89 336 403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000023.jpg</td>\n",
       "      <td>229 8 500 245 219 229 500 334 0 1 369 117 1 2 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000026.jpg</td>\n",
       "      <td>124 89 212 337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000032.jpg</td>\n",
       "      <td>77 103 183 375 87 132 123 197 179 194 229 213 ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           fn                                                bbs\n",
       "0  000012.jpg                                     96 155 270 351\n",
       "1  000017.jpg                       61 184 199 279 77 89 336 403\n",
       "2  000023.jpg  229 8 500 245 219 229 500 334 0 1 369 117 1 2 ...\n",
       "3  000026.jpg                                     124 89 212 337\n",
       "4  000032.jpg  77 103 183 375 87 132 123 197 179 194 229 213 ..."
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(MBB_CSV, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "aug_tfms = [RandomRotate(10, tfm_y = TfmType.COORD),\n",
    "           RandomLighting(0.05,0.05, tfm_y = TfmType.COORD),\n",
    "           RandomFlip(tfm_y = TfmType.COORD)]\n",
    "tfms = tfms_from_model(f_model, size, aug_tfms=aug_tfms, crop_type=CropType.NO, tfm_y = TfmType.COORD)\n",
    "md = ImageClassifierData.from_csv(PATH, JPEGS, MBB_CSV, tfms=tfms, continuous=True, num_workers=4, val_idxs=val_idx, bs=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConcatLblDataset(Dataset):\n",
    "    def __init__(self, ds, y2):\n",
    "        self.ds,self.y2 = ds,y2\n",
    "        self.sz = ds.sz\n",
    "    def __len__(self): return len(self.ds)\n",
    "    \n",
    "    def __getitem__(self, i):\n",
    "        x,y = self.ds[i]\n",
    "        return (x, (y,self.y2[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_ds2 = ConcatLblDataset(md.trn_ds, trn_cats)\n",
    "val_ds2 = ConcatLblDataset(md.val_ds, val_cats)\n",
    "md.trn_dl.dataset = trn_ds2\n",
    "md.val_dl.dataset = val_ds2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model and anchors 4x4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "anc_grid = 4\n",
    "k = 1 # for later\n",
    "anc_offset = 1/(2*anc_grid)\n",
    "anc_x = np.tile(np.linspace(anc_offset,1-anc_offset,anc_grid), anc_grid)\n",
    "anc_y = np.repeat(np.linspace(anc_offset,1-anc_offset,anc_grid), anc_grid)\n",
    "anc_ctrs = np.tile(np.stack([anc_x,anc_y], axis=1), (k,1))\n",
    "anc_sizes = np.array([[1/anc_grid,1/anc_grid] for i in range(anc_grid * anc_grid)])\n",
    "anchors = V(np.concatenate([anc_ctrs, anc_sizes], axis=1), requires_grad=False).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_sizes = V(np.array([1/anc_grid]), requires_grad=False).unsqueeze(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hw2corners(ctr, hw): return torch.cat([ctr-hw/2,ctr+hw/2], dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "anchor_cnr = hw2corners(anchors[:,:2], anchors[:,2:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Activations necessary for detecting one object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_clas = len(id2cats)+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class StdConv(nn.Module):\n",
    "    def __init__(self, n_in,n_out,stride=2,dp = 0.1):\n",
    "        super().__init__()\n",
    "        self.conv = nn.Conv2d(n_in,n_out,3,stride=stride,padding=1)\n",
    "        self.bn = nn.BatchNorm2d(n_out)\n",
    "        self.dropout = nn.Dropout(dp)\n",
    "        \n",
    "    def forward(self,x):\n",
    "        return self.dropout(self.bn(F.relu(self.conv(x))))\n",
    "    \n",
    "def flatten_conv(x,k):\n",
    "    bs,nf,gx,gy = x.size()\n",
    "    x = x.permute(0,2,3,1).contiguous()\n",
    "    return x.view(bs,-1,nf//k) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OutConv(nn.Module):\n",
    "    def __init__(self, k, n_in, bias):\n",
    "        super().__init__()\n",
    "        self.k = k\n",
    "        self.oconv1 = nn.Conv2d(n_in, n_clas * k, 3, padding=1)\n",
    "        self.oconv2 = nn.Conv2d(n_in, 4 * k, 3, padding = 1)\n",
    "        self.oconv1.bias.data.zero_().add_(bias)\n",
    "        \n",
    "    def forward(self,x):\n",
    "        return [flatten_conv(self.oconv1(x), self.k),\n",
    "                flatten_conv(self.oconv2(x), self.k)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SSD_Head(nn.Module):\n",
    "    def __init__(self, k, bias):\n",
    "        super().__init__()\n",
    "        self.drop = nn.Dropout(0.25)\n",
    "        self.conv0 = StdConv(512,256,stride=1)\n",
    "        self.conv1 = StdConv(256,256)\n",
    "        self.out = OutConv(k, 256, bias)\n",
    "    \n",
    "    def forward(self,x):\n",
    "        x = self.drop(F.relu(x))\n",
    "        x = self.conv0(x)\n",
    "        x = self.conv1(x)\n",
    "        return self.out(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Writing the loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_embedding(labels, num_classes):\n",
    "    return torch.eye(num_classes)[labels.data.cpu()]\n",
    "\n",
    "class BCE_Loss(nn.Module):\n",
    "    def __init__(self, num_classes):\n",
    "        super().__init__()\n",
    "        self.num_classes = num_classes\n",
    "    \n",
    "    def forward(self, preds, targets):\n",
    "        t = one_hot_embedding(targets, self.num_classes+1)\n",
    "        t = V(t[:,:-1].contiguous()) #bg class is predicted when none of the others go out.\n",
    "        x = preds[:,:-1]\n",
    "        w = self.get_weight(x,t)# for the last part\n",
    "        return F.binary_cross_entropy_with_logits(x, t, w, size_average=False) / self.num_classes\n",
    "    \n",
    "    def get_weight(self,x,t):\n",
    "        return None\n",
    "\n",
    "loss_f = BCE_Loss(len(id2cats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def intersection(box_a,box_b):\n",
    "    min_xy = torch.max(box_a[:,None,:2],box_b[None,:,:2])\n",
    "    max_xy = torch.min(box_a[:,None,2:],box_b[None,:,2:])\n",
    "    inter = torch.clamp(max_xy-min_xy,min=0)\n",
    "    return inter[:,:,0] * inter[:,:,1]\n",
    "\n",
    "def get_size(box):\n",
    "    return (box[:,2]-box[:,0]) * (box[:,3] - box[:,1])\n",
    "\n",
    "def jaccard(box_a,box_b):\n",
    "    inter = intersection(box_a,box_b)\n",
    "    union = get_size(box_a).unsqueeze(1) + get_size(box_b).unsqueeze(0) - inter\n",
    "    return inter/union"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Removes the zero padding in the target bbox/class\n",
    "def get_y(bbox,clas):\n",
    "    bbox = bbox.view(-1,4)/size\n",
    "    bb_keep = ((bbox[:,2] - bbox[:,0])>0.).nonzero()[:,0]\n",
    "    return bbox[bb_keep], clas[bb_keep]\n",
    "    \n",
    "def actn_to_bb(actn, anchors):\n",
    "    actn_bbs = torch.tanh(actn)\n",
    "    actn_ctrs = (actn_bbs[:,:2] * grid_sizes/2) + anchors[:,:2]\n",
    "    actn_hw = (1 + actn_bbs[:,2:]/2) * anchors[:,2:]\n",
    "    return hw2corners(actn_ctrs,actn_hw)\n",
    "\n",
    "def map_to_ground_truth(overlaps, print_it=False):\n",
    "    prior_overlap, prior_idx = overlaps.max(1)\n",
    "    if print_it: print(prior_overlap)\n",
    "#     pdb.set_trace()\n",
    "    gt_overlap, gt_idx = overlaps.max(0)\n",
    "    gt_overlap[prior_idx] = 1.99\n",
    "    for i,o in enumerate(prior_idx): gt_idx[o] = i\n",
    "    return gt_overlap,gt_idx\n",
    "\n",
    "def ssd_1_loss(b_c,b_bb,bbox,clas,print_it=False, use_ab=True):\n",
    "    bbox,clas = get_y(bbox,clas)\n",
    "    a_ic = actn_to_bb(b_bb, anchors)\n",
    "    overlaps = jaccard(bbox.data, (anchor_cnr if use_ab else a_ic).data)\n",
    "    gt_overlap,gt_idx = map_to_ground_truth(overlaps,print_it)\n",
    "    gt_clas = clas[gt_idx]\n",
    "    pos = gt_overlap > 0.4\n",
    "    pos_idx = torch.nonzero(pos)[:,0]\n",
    "    gt_clas[1-pos] = len(id2cats)\n",
    "    gt_bbox = bbox[gt_idx]\n",
    "    loc_loss = ((a_ic[pos_idx] - gt_bbox[pos_idx]).abs()).mean()\n",
    "    clas_loss  = loss_f(b_c, gt_clas)\n",
    "    return loc_loss, clas_loss\n",
    "\n",
    "def ssd_loss(pred,targ,print_it=False):\n",
    "    lcs,lls = 0.,0.\n",
    "    for b_c,b_bb,bbox,clas in zip(*pred,*targ):\n",
    "        loc_loss,clas_loss = ssd_1_loss(b_c,b_bb,bbox,clas,print_it)\n",
    "        lls += loc_loss\n",
    "        lcs += clas_loss\n",
    "    if print_it: print(f'loc: {lls.data[0]}, clas: {lcs.data[0]}')\n",
    "    return lls+lcs\n",
    "\n",
    "def ssd_loss2(pred,targ):\n",
    "    lcs,lls = 0.,0.\n",
    "    for b_c,b_bb,bbox,clas in zip(*pred,*targ):\n",
    "        loc_loss,clas_loss = ssd_1_loss(b_c,b_bb,bbox,clas,use_ab=False)\n",
    "        lls += loc_loss\n",
    "        lcs += clas_loss\n",
    "    return lls+lcs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Influence of the initial bias"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In Jeremy's notebook, the last convolutional layer that gives the categories is initiliazed with a bias of -3. When I asked him why, he explained it was because a lot of the anchors should predict the background class, so we should make it difficult for the sigmoid to be activated, hence this factor.\n",
    "\n",
    "Let's start with the bias of -3 like in Jeremy's notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_reg4 = SSD_Head(k,-3.)\n",
    "models = ConvnetBuilder(f_model, 0, 0, 0, custom_head=head_reg4)\n",
    "learn = ConvLearner(md, models)\n",
    "learn.opt_fn = optim.Adam\n",
    "learn.crit = ssd_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01576d0b52c34c1eb0217260ac4928cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 83%|███████████████████████████████████████████████████████▎           | 52/63 [00:31<00:06,  1.66it/s, loss=1.31e+03]"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEOCAYAAACEiBAqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4VGXexvHvLz2BhEBIaAFCL1ICBARcURFdZFk7KFZkFXXXsnZ9191Vd11dey9gbyCCFSsqio0Sekd6J6EkBEgCSZ73jxndLBsgQCYnM3N/rmuuzJw5Z+bOEefOmecUc84hIiLhK8LrACIi4i0VgYhImFMRiIiEORWBiEiYUxGIiIQ5FYGISJhTEYiIhDkVgYhImFMRiIiEORWBiEiYi/I6QGXUr1/fZWRkeB1DRCSozJw5c6tzLvVQ8wVFEWRkZJCdne11DBGRoGJmayozn74aEhEJcyoCEZEwpyIQEQlzKgIRkTCnIhARCXMqAhGRMBcUu48eqRmrtxNhcEzjOsRFR3odR0SkRgrpInh00jJ+XLGNqAijY+MkujVNJrNZMt2a1qV5SgJm5nVEERHPWTBcvD4rK8sdyQFlOQVFzFmbx5x1ecxem8fc9Xns2VsKQN2EaDKbJtO9WV1O69yQ1mmJVR1bRMRTZjbTOZd1yPlCuQj2V1rmWLalwF8MO5i9No/lubtwDjo1SeLMzCac3rUxaUlxVZBaRMRbKoJKyiko4qO5m/hgzgbmrc8nwuC41vU5I7MJAzs1pHZsSH97JiIhTEVwBJbn7OKDORt4f84G1m0vJC46ggEdGnBuj3T6tUklIkJjCiISPFQER8E5x6y1O3hv9gY+nreJHXv20TwlgYuObc6QrHSSE2KqLYuIyJFSEVSR4pJSPluwmdd/WkP2mh3ERkVwRmZjLu6dQef0Op5kEhGpDBVBACzauJPXp67h/dkbKNxXSmbTZC7p05xBnRvpOAURqXFUBAGUX7iPd2et5/Wpa1iZu5vkhGj6t09jQIcG9GubqgFmEakRVATVwDnHjyu28U72OiYvzSW/cB8xkREc27IeAzo04OQOaaTXTfA6poiEKRVBNSspLWPmmh18uXgLXy3OYeXW3QC0b5jIgA4N6N8hja7pyURqzyMRqSYqAo+tyN3FV4u38OXiHLJXb6fM+Y5mPqFtKie1T6Nfm1Tq1tLeRyISOCqCGmTH7r1M+TmXb5fm8s2yXLbv3kuEQWbTZE5ql8ZJ7dPo2ChJxymISJVSEdRQpWWO+Rvymbwkh2+W5jB3fT4AjerEcdlxGQzr1YzEuGiPU4pIKFARBIncgmKmLMtlwqz1/LhiG4lxUVzcuzmXHdeC1MRYr+OJSBBTEQShuevyeH7KCj5dsJnoyAiG9EhnZL+WNE+p5XU0EQlCKoIgtjJ3F6O/W8mEmRsoKStjUOdGXHVCKzo10ZHMIlJ5KoIQkLOziJd+WM2bU9dQUFxC67Ta9MyoS1bzevTMqEfTevG6uI6IHJCKIITsLNrHO9nr+WH5VrJXb2dnUQkAaYmx9MyoR1ZGXXpm1KN9w0SiInUZahHxURGEqLIyx885u5ixejvZq7czY/UONuQVAlCvVgyX9GnOpX0ydIyCiNScIjCzSCAb2OCcG2xmLYCxQD1gFnCxc27vwV5DRXBwG/MKmbF6Ox/O2chXS3KIj47kvJ5Nufz4FjrFhUgYq0lFcCOQBST5i2Ac8K5zbqyZPQfMdc49e7DXUBFU3tLNBTw/ZQUfztmIA07v2pgrT2hJ+4ZJXkcTkWpW2SII6BfKZpYO/A54wf/YgP7AeP8srwJnBjJDuGnXMJFHhmby7a0ncWmfDD5fuJmBj33HZS9PZ9rKbQTDV4EiUr0CPbL4GHArUOZ/nALkOedK/I/XA00CnCEsNUmO52+/78iPt/fnxlPaMnd9PueNmso1b80mf88+r+OJSA0SsCIws8FAjnNuZvnJFcxa4Z+oZjbSzLLNLDs3NzcgGcNBckIM153chh9u68/Np7bl84WbOe3xKUxduc3raCJSQwRyi+A44HQzW41vcLg/vi2EZDP75cot6cDGihZ2zo1yzmU557JSU1MDGDM8xMdEck3/Nky4ui8xUREMGz2VBz9fwr7SskMvLCIhLWBF4Jy7wzmX7pzLAM4HvnbOXQhMBs71z3Yp8EGgMsj/6to0mY+vO54hPdJ5evIKzn3uJ1b7r50gIuHJi6OPbgNuNLPl+MYMXvQgQ1irFRvFA+d25ekLurMqdxe/e+I7xs9cr4FkkTClA8rC3Ma8Qm54ew7TVm1ncJdG3HtWZ+rE6zTYIqGgRuw+KjVf4+R43rqiN7f8th2fLdjM4Ce/Y3nOLq9jiUg1UhEIkRHGn05qzbir+lC4t5Rznv2RadqrSCRsqAjkV92b1eW9Px5HamIsF784nQ/mbPA6kohUAxWB/Jem9RKYcFVfujVL5vqxc3h68nINIouEOBWB/I86CdG89odenNWtCQ9+vpTbJ8zX8QYiISzq0LNIOIqNiuSRoV1JrxvPk18vZ2N+Ic9c2J3EOO1RJBJqtEUgB2Rm3HRqOx44pws/rdjGkOd+YlN+odexRKSKqQjkkIb2bMrLl/Vk/Y5Cznz6BxZt3Ol1JBGpQioCqZTj26Qy/uo+RJgx9PmfmLJMJwIUCRUqAqm09g2TeO+Px5FeN54Rr8xgXPY6ryOJSBVQEchhaVgnjneu6kOfVincOn4ej05apt1LRYKcikAOW2JcNC8N78mQHuk8/tXP3DJ+nnYvFQli2n1Ujkh0ZAQPnNuFJnXjeezLn9mys0i7l4oEKW0RyBEzM/48oC0Pnvuf3Us35xd5HUtEDpOKQI7akKymvDTct3vpWc/8wOJN2r1UJJioCKRK9Gubyrgr+1DmHGc8/QPPfrOCEo0biAQFFYFUmY6Nk/jo2t/Qv10a//5sCWc986MOPhMJAioCqVJpiXE8d3EPnrmwO5vyCzn9qe955IulFJeUeh1NRA5ARSABMahzIybdcAKnd23ME18vZ/AT3zN77Q6vY4lIBVQEEjB1a8XwyHmZvDy8J7uKSzjn2R/558RFFO7V1oFITaIikIA7qX0aX9zQj2G9mvHC96sY+PgUlucUeB1LRPxUBFItEuOiufeszoy5oje7i0s5f9RUft6iMhCpCVQEUq36tEph7MjeRJhx/qipLN2sMhDxmopAql3rtNqMHdmbqEhj2OipLNmsXUxFvKQiEE+0TK3N2JF9iImMYNioqTreQMRDKgLxTIv6tRg7sjdx0ZFc8MJUFm7M9zqSSFhSEYinMvxlkBAdyQWjp7Fgg8pApLqpCMRzzVNqMXZkH2rHRnHhC9OYv15lIFKdVARSIzRLSWDsyN7+MpjKvPV5XkcSCRsqAqkxmtZL4O0re5MUH83lr2azdVex15FEwkLAisDM4sxsupnNNbOFZna3f/rJZjbLzOaY2fdm1jpQGST4pNdNYNTFWeQV7uOmcXMpK9P1kEUCLZBbBMVAf+dcVyATGGhmvYFngQudc5nAW8CdAcwgQahj4yT+Orgj3y7LZdR3K72OIxLyAlYEzmeX/2G0/+b8tyT/9DrAxkBlkOB10bHNGNS5IQ9+vpSZa7Z7HUckpAV0jMDMIs1sDpADTHLOTQMuBz4xs/XAxcD9gcwgwcnMuO/sLjROjuO6MXPI27PX60giISugReCcK/V/BZQO9DKzTsANwCDnXDrwMvBIRcua2Ugzyzaz7Nzc3EDGlBqqTnw0Tw3rTk5BEbeOn4dzGi8QCYRq2WvIOZcHfAOcBnT1bxkAvA30PcAyo5xzWc65rNTU1OqIKTVQ16bJ3DawPV8s2sKrP672Oo5ISArkXkOpZpbsvx8PDAAWA3XMrK1/tlP800QO6A+/acHJ7dP41ydLdLCZSAAEcougETDZzOYBM/CNEUwErgAmmNlcfGMEtwQwg4QAM+OhIV1JqR3DNWNmUVC0z+tIIiHFguF716ysLJedne11DPHYjNXbOX/UVAZ1bsQT52diZl5HEqnRzGymcy7rUPPpyGIJGj0z6nHjKW35aO5G3p6xzus4IiFDRSBB5eoTWnF8m/r8/cOFLNOlLkWqhIpAgkpEhPHI0EwS46K4bsxsivaVeh1JJOipCCTopCbG8uCQrizZXMD9ny7xOo5I0FMRSFA6qV0aI45rwSs/rubrJVu8jiMS1FQEErRuO60dHRolcfM788jZWeR1HJGgpSKQoBUbFcmTwzLZs7eEm97RKatFjpSKQIJa67RE/jb4GL77eSsvfr/K6zgiQUlFIEFvWK+m/PaYBjzw+RIWbNApKEQOl4pAgp6Zcf/ZXUipFct1Y2azu7jE60giQUVFICGhbq0YHj0vk1XbdnPPR4u8jiMSVFQEEjL6tErhjye24u3sdXw8b5PXcUSChopAQsqfB7Qls2kyd7w7jw15hV7HEQkKKgIJKdGRETxxfjfKHFw3Zjb7Ssu8jiRS46kIJOQ0S0ngX2d3ZuaaHTz0xVKv44jUeCoCCUmnd23Mhcc24/lvV/LVYp2CQuRgVAQSsv46uCMdGyVx0ztzNV4gchAqAglZcdGRPHNhd0pKHde8NUvjBSIHoCKQkJZRvxb3n9OZ2WvzeOAznbJapCIqAgl5g7s05uLezRn93SomLdJ4gcj+VAQSFv7yuw50apLEze/MZf2OPV7HEalRVAQSFuKiI3n6gu6UlTn+9NZs9pZovEDkFyoCCRvNU2rxwLldmLsuj39rvEDkVyoCCSundW7E8L4ZvPj9Kj5fuNnrOCI1QqWKwMyuN7Mk83nRzGaZ2amBDicSCHcMak+X9Drc8s5cNur4ApFKbxGMcM7tBE4FUoHLgPsDlkokgHyXuOxGSZnj1vHzdIlLCXuVLQLz/xwEvOycm1tumkjQaZ5Si/8b1IHvl2/ljWlrvI4j4qnKFsFMM/sCXxF8bmaJgHa7kKB24bHN6Nc2lX99sphVW3d7HUfEM5Utgj8AtwM9nXN7gGh8Xw+JBC0z44FzuhATGcFN4+ZQqq+IJExVtgj6AEudc3lmdhFwJ6CrhEvQa1gnjn+c2YlZa/N4fsoKr+OIeKKyRfAssMfMugK3AmuA1w62gJnFmdl0M5trZgvN7G7/dDOze81smZktNrPrjuo3EDlKp3dtzKDODXl00jIWb9rpdRyRalfZIihxzjngDOBx59zjQOIhlikG+jvnugKZwEAz6w0MB5oC7Z1zHYCxR5RcpIqYGf88szN14mO4cdxcHXUsYaeyRVBgZncAFwMfm1kkvnGCA3I+u/wPo/03B1wN3OOcK/PPl3NEyUWqUL1aMdx/dmcWb9rJ418t8zqOSLWqbBGch+8v/BHOuc1AE+DBQy1kZpFmNgfIASY556YBrYDzzCzbzD41szZHmF2kSg3o2IAhPdJ59psVzFq7w+s4ItWmUkXg//B/E6hjZoOBIufcQccI/MuVOucygXSgl5l1AmL9y2cBo4GXKlrWzEb6yyI7Nze3kr+OyNH52+870qhOPDePm0vh3lKv44hUi8qeYmIoMB0YAgwFppnZuZV9E+dcHvANMBBYD0zwP/Ue0OUAy4xyzmU557JSU1Mr+1YiRyUxLpoHh3Rh5dbdOjGdhI3KfjX0F3zHEFzqnLsE6AX89WALmFmqmSX778cDA4AlwPtAf/9sJwD6QlZqlL6t6jO8bwav/Lia737W1qiEvsoWQcR+g7rbKrFsI2Cymc0DZuAbI5iI7xxF55jZfOA+4PLDzCwScLcNbE/bBrX505uzWJ6z69ALiAQx8+0VeoiZzB7E9xXOGP+k84B5zrnbApjtV1lZWS47O7s63krkV+u27+GsZ34gISaK9/7Yl5TasV5HEjksZjbTPx57UJUdLL4FGIWvDLoCo6qrBES80rReAqMvyWLLziKueC2bon0aPJbQVOkL0zjnJjjnbnTO3eCcey+QoURqim7N6vLYeZnMWpvHze/M1SmrJSQdtAjMrMDMdlZwKzAzHYsvYeG0zo2447T2TJy3iYcnLfU6jkiVizrYk865Q51GQiQsjOzXktXbdvP05BU0T6nF0KymXkcSqTIHLQIR8TEz7jmjE+t3FPJ/784nPTmevq3rex1LpEro4vUilRQdGcHTF3anZWotrnxjJstzCryOJFIlVAQihyEpLpqXhvckNiqS4S/PILeg2OtIIkdNRSBymNLrJvDipVls3VXMyNez2Veq01ZLcFMRiByBrk2TeWhIV2avzeOpr5d7HUfkqKgIRI7Q4C6NOatbE56avJx56/O8jiNyxFQEIkfhrtOPIbV2LDe8PUdHHkvQUhGIHIU68b7TVq/I3c2Dn+tgMwlOKgKRo3R8m1Qu6dOcF79fxU8rtnkdR+SwqQhEqsDtp7WnRf1a3PzOXAqK9nkdR+SwqAhEqkBCTBQPD+3KpvxC/jFxkddxRA6LikCkinRvVperT2zFuOz1fLloi9dxRCpNRSBSha4/uS0dGiVx+7vz2b57r9dxRCpFRSBShWKiInhkaFd2Fu7jL+/NpzJXABTxmopApIp1aJTEDae05dMFm/lgzkav44gckopAJABG9mtJj+Z1+dsHC9iUX+h1HJGDUhGIBEBkhPHI0K6UlDn++OYsikt01LHUXCoCkQBpnlKLh/0npvv7Bws1XiA1lopAJIBO69yIP53UirEz1vHmtLVexxGpkIpAJMBuPKUdJ7ZL5e6PFpK9ervXcUT+h4pAJMAiI4zHz+9Get0ErnpjFpvzi7yOJPJfVAQi1aBOfDSjLu5B4d4Srnxjpk5ZLTWKikCkmrRpkMjDQzOZuy6Pv32wQIPHUmOoCESq0cBODbmuf2vGZa/njalrvI4jAqgIRKrdnwe05eT2adz90SKmrdT1C8R7KgKRahYRYTx6fibN6iXwp7dmsTFPRx6LtwJWBGYWZ2bTzWyumS00s7v3e/5JM9sVqPcXqcmS4qIZdUkPivaVMfL1bHYVl3gdScJYILcIioH+zrmuQCYw0Mx6A5hZFpAcwPcWqfFapyXyxLBMFm8q4IpXs7UnkXgmYEXgfH75iz/af3NmFgk8CNwaqPcWCRb92zfgoSFd+GnlNq4dM5uS0jKvI0kYCugYgZlFmtkcIAeY5JybBlwDfOic23SIZUeaWbaZZefm5gYypoinzuqWzl2/78ikRVu4/d35lJVpt1KpXlGBfHHnXCmQaWbJwHtm1g8YApxYiWVHAaMAsrKy9H+GhLThx7Ugv7CER79cRlJcNH8d3AEz8zqWhImAFsEvnHN5ZvYNcBLQGlju/0eeYGbLnXOtqyOHSE123cmtySvcy0s/rKJuQjTXntzG60gSJgJWBGaWCuzzl0A8MAD4t3OuYbl5dqkERHzMjL/+riP5hft4eNIy6iREc0mfDK9jSRgI5BZBI+BV/+BwBDDOOTcxgO8nEvQiIowHzunCzsIS/vbBQurER3NGZhOvY0mIC1gROOfmAd0OMU/tQL2/SLCKiozgqQu6Mfzl6dw0bi6JcVH0b9/A61gSwnRksUgNFBcdyehLsujQKImr35jF7LU7vI4kIUxFIFJDJcZF8+qIXtSvHcu1Y2aTX7jP60gSolQEIjVYvVoxPDGsG5vyi/i/9+br1NUSECoCkRquR/O63HhKWz6et4m3Z6zzOo6EIBWBSBC4+oRWHNc6hbs+WsjPWwq8jiMhRkUgEgQiIoxHh2ZSKyaKa8fM1gnqpEqpCESCRFpSHA8N7cqSzQXc+/Fir+NICFERiASRk9qlcflvWvD61DV8tmCz13EkRKgIRILMrQPb07lJHW6bMI8NurqZVAEVgUiQiYmK4Mlh3SgpLePPY3UNg1C1dVcx93+6hJ1FgT9+REUgEoQy6tfin2d1YsbqHTzx9XKv40gAPPjZUl74biU5O4sC/l4qApEgdVa3dM7u3oSnvv6ZqSu3eR1HqtCcdXmMm7mOy47LoHVaYsDfT0UgEsT+cUYnmqfU4toxs9mcH/i/HCXwysocf/9gAfVrx3JdNV2TQkUgEsRqxUbx3EU92FNcwpWvZ+v4ghAwfuZ65q7P5/aB7UmMi66W91QRiAS5dg0TefS8TN+Hx4R5Oh9REMsv3Me/P1tC92bJnNWt+q5DoSIQCQGnHtOQm05py/tzNjJqykqv48gReuzLZWzfs5d7zuhERET1XbNaRSASIq7p35rfdW7E/Z8tYfKSHK/jyGFaurmA135aw7BezejUpE61vreKQCREmBkPDulCh4ZJXDdmNstzdnkdSSrJOcddHy6kdmwUt5zartrfX0UgEkISYqIYfWkWMVERjHwtWxezCRKfzN/MTyu3cfOpbalbK6ba319FIBJimiTH8+xFPVi3Yw/XjplNaZkGj2uyPXtLuPfjRXRolMQFxzb3JIOKQCQE9WpRj3vO6MSUZbnc/6nOVFqTPTN5BRvzi7jnjGOIrMYB4vKiPHlXEQm4Yb2asXjTTkZ/t4r2DZM4p0e615FkP2u27WbUlJWcmdmYnhn1PMuhLQKREPbXwR3p0zKFO96bT/bq7V7Hkf38Y+IioiONOwZ18DSHikAkhEVHRvD0hd1pkhzPiFdmsEyXuawxJi/N4cvFOVx7chsaJMV5mkVFIBLi6tWK4bURvYiNjuTSl6azUdcw8NzabXu4Y8J8WtavxYjjWngdR0UgEg6a1kvg1ct6sauohEtemk7enr1eRwpbG/IKGTZ6KkUlpTx9YXdiorz/GPY+gYhUi46Nkxh1SRZrt+3hD69mU7hXJ6irbpvzi7hg9FR2Fu3j9RHH0qFRkteRABWBSFjp0yqFx87PZNbaHVw7ZpaublaNcguKueCFqWwtKObVEb3onF69p5E4GBWBSJgZ1LkRd59+DF8uzuEv7y3Q2Uqrwfbde7nohWlsyivi5ct60b1ZXa8j/ZeAHUdgZnHAFCDW/z7jnXN/N7M3gSxgHzAduNI5p+PgRarRJX0yyNlZzFOTl5OWFMtNHpzfJlzk79nHRS9MY/W23bw8vCe9Wnh3vMCBBHKLoBjo75zrCmQCA82sN/Am0B7oDMQDlwcwg4gcwE2ntuW8rKY8+fVyXv9ptddxQtLOon1c8tI0lufs4vmLe9C3dX2vI1UoYFsEzre9+cvpD6P9N+ec++SXecxsOqDDHUU8YGbce1Yntu0u5m8fLiQ5IYbfd23sdayQsbu4hMtensHCjTt59qIenNguzetIBxTQMQIzizSzOUAOMMk5N63cc9HAxcBngcwgIgcWFRnBk8O6k9W8LteNnc0rP6zyOlJI2FVcwh9encGcdXk8Mawbp3Rs4HWkgwpoETjnSp1zmfj+6u9lZp3KPf0MMMU5911Fy5rZSDPLNrPs3NzcQMYUCWvxMZG8NuJYBnRowF0fLeKfExdRpjOWHrEFG/IZ/MR3TF+1nUeGdmVQ50ZeRzqkatlryDmXB3wDDAQws78DqcCNB1lmlHMuyzmXlZqaWh0xRcJWfEwkz13Ug0v7NOeF71dxzZhZFO3TcQaHwznH6z+t5uxnfqRwXyljrujNGZnVd93hoxHIvYZSgX3OuTwziwcGAP82s8uB3wInO+e0E7NIDREZYdx1+jGk103g3k8Wk7NzGqMvyfLkQinBZmfRPm6fMI9P5m/mxHapPDykKym1Y72OVWmBPA11I+BVM4vEt+Uxzjk30cxKgDXAT2YG8K5z7p4A5hCRSjIzrujXksbJ8dwwbg7nPPcjrwzvRbOUBK+j1Vjz1udxzVuz2ZBXyO2ntWfk8S2r9cLzVcGC4WCSrKwsl52d7XUMkbAyY/V2rngtm6gI48VLe9K1abLXkWoU5xwv/7Ca+z5dTGrtWJ68oBs9mtesYwTMbKZzLutQ8+nIYhGpUM+Meky4ui/xMZGcP2oqXy7a4nWkGiN/zz6ufH0m90xcxAltU/nk+uNrXAkcDhWBiBxQq9TavHv1cbRpUJuRr2fzyKRlYX1+osK9pYyasoITH5rM10tyuPN3HRh9SRbJCcE9jqJLVYrIQaUmxjJ2ZG/ufH8BT3z1Mz+t2Mpj53ejSXK819Gqzd6SMsbOWMtTXy8np6CYE9qmcstv29GpSc05cdzR0BiBiFTae7PXc+d7C4iMMB44twsDO9X8feSPRklpGe/O3sDjX/7MhrxCerWox82ntquR5wuqSGXHCLRFICKVdla3dLo19R2FfNUbs7jw2Gb8dXBH4qIjvY5WpcrKHBPnb+KxSctYuXU3XdLrcN/ZnTm+TX38ezuGFBWBiByWjPq1GH9VXx7+YinPT1nJjNXbeXJYd9o1TPQ6WpVYtqWA68fOYfGmnbRrkMjzF/fg1I4NQrIAfqHBYhE5bDFREdwxqAOvjejF9t17Of2p73lj6pqgv7bBp/M3cebTP5BbUMzj52fy6fXH89tjGoZ0CYCKQESOQr+2qXx6fT+ObZnCne8v4LxRU/l2WW7QFUJpmeOBz5Zw9ZuzaNcwkYnX/oYzMpsE3YFhR0qDxSJy1MrKHG9OW8PTk1eweWcRHRslcdWJrRjUqSFRkTX77828PXu5buwcpizLZVivZtx1ekdio0JjzKOyg8UqAhGpMntLynh/zgae+3YFK3N306xeAiP7teTcHuk1ckB58aadXPn6TDbnF3H3GccwrFczryNVKRWBiHimrMzxxaItPPvtCuauy6N+7VhG/CaDi3o3Jyku2ut4AHw0dyO3jp9HUnwUz17Uo8ZdR7gqqAhExHPOOaau3M5z367g22W5xEVH0KdlCie0TeWEdmlkpCRUyUDs5vwipq3axtSV25m9dgcxURGkJcaSmhhHWmIsaUmxNEiMIy0pltTEWF75YTXPT1lJz4y6PH1hd9IS46rgt615VAQiUqMs3JjPO9nr+XZZLqu27gagWb0ETmibSr+2qfRtlUKt2Mrt0b4xr9D3wb9iO9NWbWP1tj0AJMZF0aN5XZyDnIJicguK2LZ7LxV9zF3Spzl3/q4jMVE1ewzjaKgIRKTGWrNtN1OW5fLtslx+XLGNPXtLiY40ejSvS1piHKVljpKyMkpKHSVl7tfHpWWOzTuLWLe9EICkuCh6tUihd8t69G6ZQodGSUTut6fPvtIytu4qJmdnMTkFxeQUFNG4Tjwnta+51xCuKioCEQkKxSWlzFzeiWH0AAAIr0lEQVS9g2+X5fL98q3sLi4hKjKCqAgjMsLK/YwgMsJIToimZ0Y9jm1Zj/YN//eDX/5Dp5gQkaAQGxVJ39b16du6vtdRwlbofjkmIiKVoiIQEQlzKgIRkTCnIhARCXMqAhGRMKciEBEJcyoCEZEwpyIQEQlzQXFksZnlAz9X8FQdIP8Q0/Z/XB/YWqUBD6yifIFa/lDzHuz5Az1XmfVb0bRgWcdVuX4PNY/WcdXMG4rrOJCfE82dc6mHfAXnXI2/AaMqO33/aRU8zvY6dyCWP9S8B3v+aNZvMK/jqly/Wsdax16s36pY3jkXNF8NfXQY0/efdqBlq8PRvvfhLH+oeQ/2/NGs38q8dyAdzXtX5fo91Dxax1Uzbyiu4+r8nKhQUHw1VJXMLNtV4iRMcuS0jgNP6zjwwmkdB8sWQVUa5XWAMKB1HHhax4EXNus47LYIRETkv4XjFoGIiJSjIhARCXMqAhGRMKci2I+Z1TKzmWY22OssocjMOpjZc2Y23syu9jpPKDKzM81stJl9YGanep0n1JhZSzN70czGe52lqoRMEZjZS2aWY2YL9ps+0MyWmtlyM7u9Ei91GzAuMCmDW1WsY+fcYufcVcBQICx2zTscVbSO33fOXQEMB84LYNygU0Xrd6Vz7g+BTVq9QmavITPrB+wCXnPOdfJPiwSWAacA64EZwDAgErhvv5cYAXTBd1h5HLDVOTexetIHh6pYx865HDM7HbgdeMo591Z15Q8GVbWO/cs9DLzpnJtVTfFrvCpev+Odc+dWV/ZACpmL1zvnpphZxn6TewHLnXMrAcxsLHCGc+4+4H+++jGzk4BaQEeg0Mw+cc6VBTR4EKmKdex/nQ+BD83sY0BFUE4V/Ts24H7gU5XAf6uqf8OhJmSK4ACaAOvKPV4PHHugmZ1zfwEws+H4tghUAod2WOvYzE4EzgZigU8Cmix0HNY6Bq4FBgB1zKy1c+65QIYLAYf7bzgFuBfoZmZ3+AsjqIV6EVgF0w75XZhz7pWqjxKyDmsdO+e+Ab4JVJgQdbjr+AngicDFCTmHu363AVcFLk71C5nB4gNYDzQt9zgd2OhRllCldRx4WseBFfbrN9SLYAbQxsxamFkMcD7woceZQo3WceBpHQdW2K/fkCkCMxsD/AS0M7P1ZvYH51wJcA3wObAYGOecW+hlzmCmdRx4WseBpfVbsZDZfVRERI5MyGwRiIjIkVERiIiEORWBiEiYUxGIiIQ5FYGISJhTEYiIhDkVgVQ5M9tVDe9xeiVPK16V73mimfU9guW6mdkL/vvDzeypqk93+MwsY//TMVcwT6qZfVZdmcQbKgKpsfynB66Qc+5D59z9AXjPg51/60TgsIsA+D/gySMK5DHnXC6wycyO8zqLBI6KQALKzG4xsxlmNs/M7i43/X3/leAWmtnIctN3mdk9ZjYN6GNmq83sbjObZWbzzay9f75f/7I2s1fM7Akz+9HMVprZuf7pEWb2jP89JprZJ788t1/Gb8zsX2b2LXC9mf3ezKaZ2Wwz+9LMGvhPXXwVcIOZzTGz4/1/LU/w/34zKvqwNLNEoItzbm4FzzU3s6/86+YrM2vmn97KzKb6X/OeirawzHclvY/NbK6ZLTCz8/zTe/rXw1wzm25mif6//L/zr8NZFW3VmFmkmT1Y7r/VleWefh+4sML/wBIanHO66ValN2CX/+epwCh8Z3eMACYC/fzP1fP/jAcWACn+xw4YWu61VgPX+u//EXjBf384vgvbALwCvON/j474zi0PcC6+U11HAA2BHcC5FeT9Bnim3OO6/Oeo+8uBh/337wJuLjffW8Bv/PebAYsreO2TgAnlHpfP/RFwqf/+COB9//2JwDD//at+WZ/7ve45wOhyj+sAMcBKoKd/WhK+MwwnAHH+aW2AbP/9DGCB//5I4E7//VggG2jhf9wEmO/1vyvdAncL9dNQi7dO9d9m+x/XxvdBNAW4zszO8k9v6p++DSgFJuz3Ou/6f87Edy2DirzvfNePWGRmDfzTfgO845++2cwmHyTr2+XupwNvm1kjfB+uqw6wzACgo9mvZzFOMrNE51xBuXkaAbkHWL5Pud/ndeCBctPP9N9/C3iogmXnAw+Z2b+Bic6578ysM7DJOTcDwDm3E3xbD8BTZpaJb/22reD1TgW6lNtiqoPvv8kqIAdofIDfQUKAikACyYD7nHPP/9dE38VpBgB9nHN7zOwbfJcHBShyzpXu9zrF/p+lHPjfbHG5+7bfz8rYXe7+k8AjzrkP/VnvOsAyEfh+h8KDvG4h//ndDqXSJ/5yzi0zsx7AIOA+M/sC31c4Fb3GDcAWoKs/c1EF8xi+La/PK3guDt/vISFKYwQSSJ8DI8ysNoCZNTGzNHx/be7wl0B7oHeA3v974Bz/WEEDfIO9lVEH2OC/f2m56QVAYrnHX+A7ayUA/r+497cYaH2A9/kR3ymPwfcd/Pf++1PxffVDuef/i5k1BvY4597At8XQHVgCNDaznv55Ev2D33XwbSmUARfjuxbv/j4HrjazaP+ybf1bEuDbgjjo3kUS3FQEEjDOuS/wfbXxk5nNB8bj+yD9DIgys3nAP/B98AXCBHwXHVkAPA9MA/IrsdxdwDtm9h2wtdz0j4CzfhksBq4DsvyDq4uo4KpVzrkl+C4Zmbj/c/7lL/Ovh4uB6/3T/wzcaGbT8X21VFHmzsB0M5sD/AX4p3NuL3Ae8KSZzQUm4ftr/hngUjObiu9DfXcFr/cCsAiY5d+l9Hn+s/V1EvBxBctIiNBpqCWkmVlt59wu811ndjpwnHNuczVnuAEocM69UMn5E4BC55wzs/PxDRyfEdCQB88zBd/F3Hd4lUECS2MEEuommlkyvkHff1R3Cfg9Cww5jPl74BvcNSAP3x5FnjCzVHzjJSqBEKYtAhGRMKcxAhGRMKciEBEJcyoCEZEwpyIQEQlzKgIRkTCnIhARCXP/D6ZTN2Ysj9eFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 1e-2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7507c1e1ec543e391defd79fc83b892",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      20.929501  18.910183 \n",
      "    1      17.86262   16.488575                                                                                        \n",
      "    2      16.310839  15.498018                                                                                        \n",
      "    3      15.109892  15.010017                                                                                        \n",
      "    4      14.098554  14.523777                                                                                        \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[14.523777]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lr, 1, cycle_len=5, use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's fo the same thing without initializing the bias at -3."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('bias-3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_reg4 = SSD_Head(k,0.)\n",
    "models = ConvnetBuilder(f_model, 0, 0, 0, custom_head=head_reg4)\n",
    "learn = ConvLearner(md, models)\n",
    "learn.opt_fn = optim.Adam\n",
    "learn.crit = ssd_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b625343e98d4b7ab93e482b4677ad52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 87%|██████████████████████████████████████████████████████████████▊         | 55/63 [00:30<00:04,  1.82it/s, loss=972]"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEOCAYAAACTqoDjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4VVXa9/HvnQKhhNASOgak9xIQ7F1HHREL6mAv2Mc2Ojq+80z10WdsYwMFUdSx4GBH7A0YQQhINUhHqQkgvSbc7x9nM5PBAwTMzs5Jfp/rOlf2Wbuc+ywxd/Zaa69l7o6IiMiekqIOQEREyiclCBERiUsJQkRE4lKCEBGRuJQgREQkLiUIERGJSwlCRETiUoIQEZG4lCBERCQuJQgREYkrJeoAfo769et7dnZ21GGIiCSUKVOmrHb3zP0dl9AJIjs7m9zc3KjDEBFJKGa2pCTHqYlJRETiUoIQEZG4lCBERCQuJQgREYlLCUJEROJSghARkbgqZYLYvL2Q92euYH7+RnYW7Yo6HBGRcimhn4M4WHkrNnDdS1MBSE02WtSvQeusdFpl1aRNg3RaN6hJdr0aVEmplPlTRASopAmiU5MMRt90JPPyNzJ31SbmrdrE7OXrGTNrBe6xY1KTjbYN0+ncJINOTTLo0qQ2bRrWpGpKcrTBi4iUkUqZINJSk+kU/OIvbtvOIhYWbGZe/kbmrNzIrGXrGTNzJa9M+gH476TRtWltcrLrcmhmDcwsiq8hIhIq891/MiegnJwcD3uqDXdn6Y9bmblsPTOWrmfWsvXMWLqODdsKAahbowo5h9Shd4u65GTXpWPjWqQmq2lKRMovM5vi7jn7O65S3kEcCDOjWd3qNKtbndM6NwJiSWPR6s1MXryWSYt+JHfJWj76dhUA1VKT6d68Nke0qs/x7bJo1zBddxgikpB0B1FKVm3YRu7iH4OksZZvV2wAoEntahzfLovj22fRt2U90lLVhyEi0SrpHYQSREjyN2zj8+/y+SQvn/HzVrN1ZxHVqyRzZKv6nNA+ixPbN6BezapRhykilVDkCcLM0oCxQFViTVmj3P0PZjYCOAZYHxx6mbtPs1g7zKPAacCWoHzqvj6jPCeI4rbtLGLCwjV8lpfPp3mrWL5+G8lJRt+W9TijSyNO6diQOjWqRB2miFQS5SFBGFDD3TeZWSowHrgZuBYY7e6j9jj+NOAmYgniMOBRdz9sX5+RKAmiOHfn2xUbGDNzBaNnrGDJmi2kJBlHtKrPGV0acXLHhmRUS406TBGpwCLvpPZY5tkUvE0NXvvKRv2AF4LzJppZbTNr5O4rwooxCmZGx8YZdGycwW9ObsusZRsYPXM5o6ev4I5RM/jdmzM5unUmA3o144R2WaRoRJSIRCTUUUxmlgxMAVoBT7r712Z2HXCvmf0P8Clwl7tvB5oAPxQ7fWlQVqESRHFmRuemGXRumsFdp7Zj2g/reG/GCt6dsZxPX8ynQa2qXNCrORf0bkajjGpRhysilUyZdFKbWW3gTWJNSGuAlUAVYCiwwN3/bGbvAfe5+/jgnE+BO919yh7XGgQMAmjevHnPJUtKtHJeQiks2sVnc/J56evvGTuvAANOaN+AgYc15+jWmSQladisiBy8yPsgfvJBZn8ANrv7g8XKjgV+4+5nmNnTwBfu/kqw7zvg2H01MSViH8SB+n7NFl6Z/D2vTf6BNZt30KxuNS7s3ZwLezVXx7aIHJSSJojQGrjNLDO4c8DMqgEnAnPMrFFQZsBZwKzglHeASyymD7C+ovU/HIzm9arz21Pb8dXdx/P4hd1pUrsaf/vgO/re/yn3vDmTBQWb9n8REZGDEGYfRCPg+aAfIgl4zd1Hm9lnZpYJGDCN2KgmgDHERjDNJzbM9fIQY0s4VVOS+WXXxvyya2PmrNzAs+MX8c/cpbz09fcc3y6Lq45sQd9D6+mpbREpNXpQLoEVbNzOPyYu4R8Tl7Bm8w7aNUznyiNbcGa3xpp1VkT2qtz1QYShsieI3bbtLOLtacsYPn4Rc1dtomGtNG49qTXn9mxGsjq0RWQPShCVkLszbt5qHv54LtN+WEebBjW56xftOK5tlpqeROTfIu+klrJnZhzdJpM3rz+cwQN7sKNwF1eMyOWCoROZ9sO6qMMTkQSjBFEBmRmndW7Ex7cdw5/7dWR+/ibOevJf3PDyVJas2Rx1eCKSINTEVAls2l7I0LELGTZ2ITuLdnFx30O45cQ2mvNJpJJSE5P8W82qKdx2Uhu+vONYzstpxoivFnP8g1/w2uQf2LUrcf9AEJFwKUFUIlm10rjv7M68e+ORZNevwZ2vz6D/kK+Yrv4JEYlDCaIS6tQkg1HX9uXhAV1Z9uNWzhr8L347agZrNm2POjQRKUeUICopM+PsHk35/DfHcNWRLXh96lKOe/ALRvxrEYVFu6IOT0TKASWISi49LZV7Tu/AB7ccRZemtfnju99yzpCvNMeTiChBSEyrrHRevLI3j1/YnSVrt3D6Y+N4/qvF6sQWqcSUIOTfzIxfdm3Mh7ccTZ+W9fjDO7O59LlJrFi/NerQRCQCShDyEw1qpfHcZb24t38nchf/yCmPjOXtactI5GdmROTAKUFIXGbGwMMO4f2bj6JVVk1ufnUaN77yDeu27Ig6NBEpI0oQsk/Z9Wvw2jV9ueOUtnw0eyUnPzKWr+avjjosESkDYa4ol2Zmk8xsupnNNrM/BeUvmdl3ZjbLzJ41s9Sg/FgzW29m04LX/4QVmxyYlOQkbjiuFW/dcAS1qqUycPjXPPrJPIrUgS1SoYV5B7EdON7duwLdgFODpURfAtoBnYFqwFXFzhnn7t2C159DjE0OQsfGGbx9wxH079aERz6Zy6XPTmK1Hq4TqbBCSxAes3swfWrwcncfE+xzYBLQNKwYpPTVqJrCQwO6cv/ZnZm8eC2nPTqOiQvXRB2WiIQg1D4IM0s2s2lAPvCxu39dbF8qcDHwQbFT+gZNUu+bWccwY5ODZ2Zc0Ls5b91wBDWrpvCrYRN58vP5emZCpIIJNUG4e5G7dyN2l9DbzDoV2z0YGOvu44L3U4FDgiapx4G34l3TzAaZWa6Z5RYUFIQZvuxH+0a1eOemIzm9S2Me+PA7Lh8xmbWbNcpJpKIok1FM7r4O+AI4FcDM/gBkArcVO2bD7iYpdx8DpJpZ/TjXGuruOe6ek5mZWRbhyz7UrJrCYxd0469ndWLCgjWc/tg4Zi9fH3VYIlIKwhzFlGlmtYPtasCJwBwzuwo4BbjQ3XcVO76hBQsnm1nvIDY1bicAM+OiPofwxvWHAzDgqQl8OVd3dyKJLsw7iEbA52Y2A5hMrA9iNPAU0ACYsMdw1nOBWWY2HXgMuMD16G5C6dQkgzevP4Lm9WpwxYjJjJz8fdQhicjPoCVHpdRt3LaT61+ayrh5q/n18a249aQ2BDeHIlIOaMlRiUx6WirPXtaL83o25bHP5nP7P6ezo1BrTIgkmpSoA5CKKTU5ib+d24WmdarzyCdzyd+wncEX9aBWWmrUoYlICekOQkJjZtx8YmsePK8rExeuYcBTEzR1uEgCUYKQ0J3bsynPXd6LpT9u5ezBX7Fo9eaoQxKRElCCkDJxVOtMXrumL9sLd3H+0xOYn78x6pBEZD+UIKTMdGhci1cH9WGXwwVDJ/LdSiUJkfJMCULKVJsG6Yy8pg/JScYFQyfoqWuRckwJQsrcoZk1GTmoL9VSk/nVsK+ZuVRJQqQ8UoKQSGTXr8HIa/qSnpbCr56ZyDff/xh1SCKyByUIiUyzutUZeU1f6taowsXDJ5G7eG3UIYlIMUoQEqkmtasxclBfstKrcsmzk7T4kEg5ogQhkWuYkcar1/ShSe1qXDliMrOWqU9CpDxQgpByISs9jX9cdRi1q1fhsucmsWSNHqYTiZoShJQbDWql8fwVvSna5Vzy7CRWb9oedUgilZoShJQrrbJqMvyyXqzasI3Ln5vMpu2FUYckUmkpQUi506N5HQYP7MG3KzZw3T+maKpwkYiEueRomplNMrPpZjbbzP4UlLcws6/NbJ6ZjTSzKkF51eD9/GB/dlixSfl3fLsG3Hd2Z8bNW82do6aza1fiLmwlkqjCvIPYDhzv7l2BbsCpZtYH+D/gEXdvDfwIXBkcfyXwo7u3Ah4JjpNKbEBOM+44pS1vTVvOfe/nRR2OSKUTWoLwmE3B29Tg5cDxwKig/HngrGC7X/CeYP8JpnUqK73rjz2Uyw7PZti4RQwbuzDqcEQqlVD7IMws2cymAfnAx8ACYJ277+55XAo0CbabAD8ABPvXA/XCjE/KPzPj92d04PTOjbh3TB5vT1sWdUgilUaoCcLdi9y9G9AU6A20j3dY8DPe3cJPGp7NbJCZ5ZpZbkFBQekFK+VWcpLx8PldOaxFXe4YNYOpmrdJpEyUySgmd18HfAH0AWqb2e61sJsCy4PtpUAzgGB/BvCTyXncfai757h7TmZmZtihSzlRNSWZpy7qSaOMNAa9MIVl67R0qUjYwhzFlGlmtYPtasCJQB7wOXBucNilwNvB9jvBe4L9n7m7hq7Iv9WpUYXhl+awfWcRVz2fy2Y9IyESqjDvIBoBn5vZDGAy8LG7jwZ+C9xmZvOJ9TEMD44fDtQLym8D7goxNklQrbLSefxX3flu5QZuHTlNw19FQmSJ/Ed6Tk6O5+bmRh2GRODZ8Yv48+hvueG4Q7njlHZRhyOSUMxsirvn7O+4lP0dIFIeXX5ENvPyN/Hk5wtolVWT/t2bRh2SSIWjqTYkIZkZf+7XkT4t6/Lb12cyZYlGNomUNiUISVipyUkMGRgb2XTNi7ks/XFL1CGJVChKEJLQ/jOyaZdGNomUMiUISXitstJ5YmAP5q7ayJ2vzyCRB16IlCdKEFIhHNMmkztOacd7M1bw7L8WRx2OSIWgBCEVxrXHtOTkDg343zF5TFr0k4fwReQAKUFIhWFmPDigK83rVueGl6eSv2Fb1CGJJDQlCKlQaqWl8tRFPdm0rZAbX/6GnUVajU7kYClBSIXTtmE695/TmUmL13L/+3OiDkckYSlBSIXUr1sTLjs8m+HjFzF6xvL9nyAiP6EEIRXW705rT4/mtblz1Azm52+MOhyRhKMEIRVWlZQkBg/sSfUqyVzz4hQ26SE6kQOiBCEVWsOMNB6/sAeL12zhzlHT9RCdyAFQgpAKr++h9fjtqW0ZM3Mlz+khOpESC3NFuWZm9rmZ5ZnZbDO7OSgfaWbTgtdiM5sWlGeb2dZi+54KKzapfK4+qiUnBQ/RaU1rkZIJ8w6iELjd3dsTW4v6BjPr4O7nu3s3d+8GvA68UeycBbv3ufu1IcYmlYyZ8eC5XWmYkcZNL3/Dui07og5JpNwLLUG4+wp3nxpsbyS2HnWT3fvNzIABwCthxSBSXEb1VAYP7EHBxu3c/tp0LVcqsh9l0gdhZtlAd+DrYsVHAavcfV6xshZm9o2ZfWlmR5VFbFK5dGlam3tOb8+nc/IZOm5h1OGIlGuhJwgzq0msKekWd99QbNeF/Pfdwwqgubt3B24DXjazWnGuN8jMcs0st6CgIMzQpYK6pO8hnN65EQ98+J0m9RPZh1AThJmlEksOL7n7G8XKU4CzgZG7y9x9u7uvCbanAAuANnte092HunuOu+dkZmaGGb5UUGbG/ed0plmdatz0ylRWb9oedUgi5VKYo5gMGA7kufvDe+w+EZjj7kuLHZ9pZsnBdkugNaA2AAlFeloqTw7swY9bdnLryGkUqT9C5CfCvIM4ArgYOL7Y0NXTgn0X8NPO6aOBGWY2HRgFXOvuuv+X0HRsnMGfzuzIuHmrefLz+VGHI1LupIR1YXcfD9he9l0Wp+x1Ys1RImXmgl7NmLRoLY98Mpeeh9ThiFb1ow5JpNzQk9RSqZkZfz2rEy3r1+DmV79hlRYZEvk3JQip9GpUTWHIRT3ZvL2Im175hkItMiQCKEGIANCmQTr39u/EpEVrefCjuVGHI1IulChBmNnNZlbLYoab2VQzOzns4ETK0tk9mnJh7+Y89eUCPvl2VdThiESupHcQVwQPuZ0MZAKXA/eHFpVIRP7wyw50bFyL216bxg9rt0QdjkikSpogdo9GOg14zt2ns5cRSiKJLC01mSEDe+LADS9PZXthUdQhiUSmpAliipl9RCxBfGhm6YB68qRCal6vOg+e15UZS9fz19F5UYcjEpmSJogrgbuAXu6+BUgl1swkUiGd0rEhg45uyYsTl/D2tGVRhyMSiZImiL7Ad+6+zswuAv4fsD68sESid8cpbemVXYe735jJ/PyNUYcjUuZKmiCGAFvMrCtwJ7AEeCG0qETKgdTkJB6/sAfVUpO57h9T2bKjMOqQRMpUSRNEocdWe+8HPOrujwLp4YUlUj40zEjj0Qu6M79gE3e/MZPY/wYilUNJE8RGM7ub2OR77wWzrqaGF5ZI+XFk6/rcflIb3p62nGfGLYo6HJEyU9IEcT6wndjzECuJLR36QGhRiZQzNxzXitM6N+S+9/MYO1cLVUnlUKIEESSFl4AMMzsD2Obu6oOQSsPMeODcrrRpkM6NL09l8erNUYckErqSTrUxAJgEnAcMAL42s3PDDEykvKlRNYVhl+SQnGRc9UIuG7ftjDokkVCVtInpHmLPQFzq7pcAvYHf7+sEM2tmZp+bWZ6ZzTazm4PyP5rZsjiLCGFmd5vZfDP7zsxOOdgvJRKWZnWr8+SverBo9WZuHTmdXVqJTiqwkiaIJHfPL/Z+TQnOLQRud/f2QB/gBjPrEOx7xN27Ba8xAMG+C4COwKnA4N1LkIqUJ4e3qs/vT2/PJ3mr+PsnmvlVKq6Srij3gZl9yH+WCT0fGLOvE9x9BbAi2N5oZnnEOrf3ph/wqrtvBxaZ2XxidyoTShijSJm59PBsZi/fwGOfzad9o1r8onOjqEMSKXUl7aS+AxgKdAG6AkPd/bcl/RAzywa6A18HRTea2Qwze9bM6gRlTYAfip22lH0nFJHImBl/7d+J7s1rc/s/p5O3YkPUIYmUuhIvGOTur7v7be5+q7u/WdLzzKwmsbWmbwmmDB8CHAp0I3aH8dDuQ+N9bJzrDTKzXDPLLSjQcEOJTtWUZJ6+qCfpaSlc/UIuazfviDokkVK1zwRhZhvNbEOc10Yz2++fTGaWSiw5vOTubwC4+yp3L3L3XcAwYs1IELtjaFbs9KbA8j2v6e5D3T3H3XMyMzNL9i1FQpJVK42nL84hf+N2rn9pCjsKNcmxVBz7TBDunu7uteK80t291r7ONTMDhgN57v5wsfLijbX9gVnB9jvABWZW1cxaAK2JDa0VKde6NavN/53TmYkL1/I/b8/SdBxSYZS0k/pgHEFsao6ZZjYtKPsdcKGZdSPWfLQYuAbA3Web2WvAt8RGQN3g7lqtRRJC/+5NWZC/mSc+n0+rrJpcdVTLqEMS+dlCSxDuPp74/Qp7Hf3k7vcC94YVk0iYbjupDQtXb+LeMXm0qF+DE9o3iDokkZ+lxJ3UIrJvSUnGQ+d1o1PjDH79yjca2SQJTwlCpBRVq5LMsEtyqJmWwlXP51KwcXvUIYkcNCUIkVLWMCONZy7pxZrN2xn0Yi7bdqorTRKTEoRICDo3zeDv53fjm+/XceeoGRrZJAlJCUIkJKd2asQdp7TlnenLefyz+VGHI3LAwhzmKlLpXX/soSzI38TDH88lu34NzuzaOOqQREpMCUIkRGbGfed0ZumPW/nNa9PJrFmVvofWizoskRJRE5NIyKqmxEY2HVKvOoNezGXOSg1/lcSgBCFSBjKqpzLiit5Ur5LMZc9OZvm6rVGHJLJfShAiZaRJ7WqMuLw3m7cXctlzk1i/VUuWSvmmBCFShto3qsXTF/dk0erNDHohl+2FekZCyi8lCJEydnir+jx4Xle+XrSW217TutZSfmkUk0gE+nVrwqoN2/jfMXNoWCuN35/RYf8niZQxJQiRiFx9VEuWr9vG8PGLaJSRpinCpdxRghCJiJnx+zM6kL9xG399L4/a1atwbs+mUYcl8m9KECIRSk4yHh7QjY3bcrlz1HRSk41+3ZpEHZYIEGIntZk1M7PPzSzPzGab2c1B+QNmNsfMZpjZm2ZWOyjPNrOtZjYteD0VVmwi5UlaajJDL86hd4u63PbadMbMXBF1SCJAuKOYCoHb3b090Ae4wcw6AB8Dndy9CzAXuLvYOQvcvVvwujbE2ETKlWpVkhl+aS+6N6vNr1/5ho9mr4w6JJHwEoS7r3D3qcH2RiAPaOLuH7l7YXDYRECNriJAjaopPHd5Lzo2yeCGl6fy+Xf5UYcklVyZPAdhZtlAd+DrPXZdAbxf7H0LM/vGzL40s6PKIjaR8iQ9LZUXLu9NmwbpXPPiFMbPWx11SFKJhZ4gzKwm8Dpwi7tvKFZ+D7FmqJeCohVAc3fvDtwGvGxmteJcb5CZ5ZpZbkFBQdjhi5S5jOqp/OPKw2hZvwZXvTCZiQvXRB2SVFKhJggzSyWWHF5y9zeKlV8KnAEM9GCpLXff7u5rgu0pwAKgzZ7XdPeh7p7j7jmZmZlhhi8SmTo1qvCPqw6jaZ3qXDFiMlOWrI06JKmEwhzFZMBwIM/dHy5WfirwW+BMd99SrDzTzJKD7ZZAa2BhWPGJlHf1a1bl5asOo0GtNC59VncSUvbCvIM4ArgYOL7Y0NXTgCeAdODjPYazHg3MMLPpwCjgWnfXn01SqWXVSuOVq/vQMCONS5+dxKd5q6IOSSoRS+TF1HNycjw3NzfqMERCt3bzDi59dhJ5Kzbw0ICuephOfhYzm+LuOfs7TrO5iiSAujWq8PLVh9HzkDrcMnIaL05cEnVIUgkoQYgkiPS0VJ6/ojcntMvi92/N4snP55PILQBS/ilBiCSQtNRkhlzUk7O6NeaBD7/j/vfnKElIaDRZn0iCSU1O4uEB3ahVLZWnxy5k/dad3Nu/M8lJFnVoUsEoQYgkoKQk409ndiSjWiqPfzaf9Vt38sj53UhLTY46NKlAlCBEEpSZcfvJbcmolspf38sjf+PXDLskh7o1qkQdmlQQ6oMQSXBXHdWSwQN7MHPZes4Z8hVL1myOOiSpIJQgRCqA0zo34uWrDmPdlh30H/wVU7//MeqQpAJQghCpIHKy6/LG9UeQnpbChUMn8sEsrSkhP48ShEgF0qJ+Dd647nA6NK7FdS9NYfj4RVGHJAlMCUKkgqlXsyqvXN2HUzo05C+jv+WP78ymaJeelZADpwQhUgGlpSbz5MAeXHlkC0Z8tZirX8hl/dadUYclCUYJQqSCSk4yfn9GB/7SryNj5xZw5hPjmbNyw/5PFAkoQYhUcBf3zebVQX3YuqOI/k9+xdvTlkUdkiQIJQiRSiAnuy6jbzqSTk1qcfOr0/jL6G/ZWbQr6rCknAtzRblmZva5meWZ2Wwzuzkor2tmH5vZvOBnnaDczOwxM5tvZjPMrEdYsYlURlm10nj56j5cdng2w8cvYuAzX1OwcXvUYUk5FuYdRCFwu7u3B/oAN5hZB+Au4FN3bw18GrwH+AWxZUZbA4OAISHGJlIppSYn8cczO/L387sxY+k6znh8nB6qk70KLUG4+wp3nxpsbwTygCZAP+D54LDngbOC7X7ACx4zEahtZo3Cik+kMjurexPeuO4IqqYkc/7TE3hx4hJNGy4/USZ9EGaWDXQHvgYauPsKiCURICs4rAnwQ7HTlgZlIhKCDo1r8e6NR3Jkq/r8/q1Z3P7adLbuKIo6LClHQk8QZlYTeB24xd33NcYu3mT2P/mTxswGmVmumeUWFBSUVpgilVJG9VSGX9qL205qw5vTltF/8L9YvFqT/UlMqAnCzFKJJYeX3P2NoHjV7qaj4Gd+UL4UaFbs9KbA8j2v6e5D3T3H3XMyMzPDC16kkkhKMn59QmtGXN6blRu28csnxvPxt6uiDkvKgTBHMRkwHMhz94eL7XoHuDTYvhR4u1j5JcFopj7A+t1NUSISvmPaZDL6piNpUb8GV7+Qy98+mEOhhsJWamHeQRwBXAwcb2bTgtdpwP3ASWY2DzgpeA8wBlgIzAeGAdeHGJuIxNG0TnVeu6YvF/ZuzuAvFnDJs5NYvUlDYSsrS+SRCzk5OZ6bmxt1GCIV0mu5P/D7t2ZRp3oVHjm/G30PrRd1SFJKzGyKu+fs7zg9SS0icQ3Iacbr1x1OtSrJ/OqZidz73rdsL9Qop8pECUJE9qpTkwze+/WRDDysOcPGLaLfE/8ib4Um/KsslCBEZJ+qV0nhr2d15rnLerF60w76PfEvnv5ygdaYqASUIESkRI5rl8VHtx7Nce0yue/9Ofxq2ESW/rgl6rAkREoQIlJidWtU4amLevLAuV2YvXwDv/j7OEZNWappOiooJQgROSBmxnk5zXj/5qNo1yid3/xzOhcPn6QnsCsgJQgROSjN6lbn1UF9+Uu/jkz/YR0n/30sj306TyOdKhAlCBE5aMlJxsV9s/n09mM4uUMDHv54Lr94dBwTFqyJOjQpBUoQIvKzZdVK44lf9WDE5b3YWbSLC4dN5LbXprFGT2EnNCUIESk1x7bN4qNbjuGG4w7l3enLOeHhL3l10vfs0pDYhKQEISKlqlqVZO44pR1jfn0UbbLSueuNmfQf8hXTf1gXdWhygJQgRCQUrRukM/KaPjxyfleWr9vKWYP/xV2vz1CzUwJRghCR0JgZ/bs35bPbj+GqI1swaspSjnvwC16YsFhTiScAJQgRCV16Wir3nN6BD245ii5Na/M/b8/mjMfHM2nR2qhDk31QghCRMtMqK50Xr+zNkIE92LB1JwOensCNL09lyRo9ZFcepUQdgIhULmbGLzo34ti2WQz5cgHDxi7kw9krGXjYIdx0fCvq1awadYgSCHPJ0WfNLN/MZhUrG1lsdbnFZjYtKM82s63F9j0VVlwiUj5Uq5LMbSe14cs7juXcns14ceISjnngC574bB5bdhRGHZ4Q4opyZnY0sAl4wd07xdn/ELF1p/9sZtnA6HjH7YtWlBOpOObnb+JvH8zho29XkZVelVtPasN5PZuSkqyW8NIW+Ypy7j4WiNsDZWYGDABeCevzRSSxtMqqydBLchh1bV+a1a3O3W/M5NRHx/H+zBV60C4pmuZhAAANq0lEQVQiUaXmo4BV7j6vWFkLM/vGzL40s6P2dqKZDTKzXDPLLSgoCD9SESlTOdl1GXVtX56+uCfuznUvTeX0x8fz0eyVmla8jIXWxASxvgXiNB2Z2RBgvrs/FLyvCtR09zVm1hN4C+jo7vtc21BNTCIVW9Eu593py3n003ksWr2Zzk0yuO2kNhzbNpNYQ4QcjMibmPbGzFKAs4GRu8vcfbu7rwm2pwALgDZlHZuIlC/JScZZ3Zvw8a1H88C5XVi3dQeXj5hM/8FfMXZuge4oQhZFE9OJwBx3X7q7wMwyzSw52G4JtAYWRhCbiJRDKclJnJfTjM9uP5b7z+5MwcbtXPLsJAY8PYEpS/SwXVjCHOb6CjABaGtmS83symDXBfy0c/poYIaZTQdGAde6u/6ri8h/SU1O4oLezfnsN8fwl7M6sWTNFs4ZMoHrX5qiFe1CEGofRNjUByFSuW3ZUciwsYt4euwCdhbt4uI+2dx0fCvq1KgSdWjlWrntgxARKS3Vq6Rw84mt+eI3x3Juz6aM+GoRxzzwOcPGLtTSp6VACUJEEl5WrTTuO7sL7998ND0OqcO9Y/I44aEveXvaMs0a+zOoiUlEKpxx8wq497085qzcSKOMNAYe1pwLejenvuZ5AkrexKQEISIVUtEu57M5+bwwYTHj5q2mSnISZ3RpxKWHZ9O1We2owzto67bsYMgXC2hatzoX9znkoK5R0gSh2VxFpEJKTjJO6tCAkzo0YH7+Jl6csJhRU5byxjfL6NqsNpcdfgindW5E1ZTkqEMtkW07ixjx1WIGfz6fjdsLufzwFqF/pu4gRKTS2LhtJ29MXcbzExazsGAzGdVSOaVjA07v0pjDD61HajmcGLCwaBdvTF3Gwx/PZeWGbRzXNpM7T21H+0a1DvqaamISEdkLd2f8/NW8+c0yPp69io3bC6ldPZVTOjTk9C6N6FsOkoW780lePn/7YA7z8jfRtVlt7jq1HX0Prfezr60mJhGRvTAzjmqdyVGtM9leWMS4uat5b+YK3pu5gpG5P1CneiqndmrIL7s05rCW9UhOKrt5n7bsKGT8vNUMHbuQ3CU/0rJ+DYYM7MGpnRqW+fxTuoMQEQls21nE2LkFvDdzBZ98u4rNO4poWCuNM7s15qxuTWjfKD2UX9I/rN3CZ3Py+XROPhMXrmFH4S4y06tyy4mtGZDTrNTvZtTEJCLyM2zbWcQneat465tlfPFdAYW7nLYN0unXvTH9ujWhSe1q+73Grl3Ozl272FnkFBbtYkfRLgqLnJ1Fu1i+bhtffJfPZ3PymZe/CYAW9WtwXNssTmifRa/sulRJCaeZSwlCRKSUrN28g/dmLOfNb5Yx9ft1APTKrkO9GlXZsrOIrTsK2by9iK07i9iyo5AtwXbhfhY6Sk02ereoy3Ftszi+XRYtM2uWxddRghARCcOSNZt5e9pyPpi1kp1Fu6heNYXqqcnUqJpMtSop1KiSTPUqKVSrkkRq8u6XkZqcREpyEqlJse2Maqkc1rIu6WmpZf4dlCBERCQuTdYnIiI/ixKEiIjEFeaCQc+aWb6ZzSpW9kczW2Zm04LXacX23W1m883sOzM7Jay4RESkZMK8gxgBnBqn/BF37xa8xgCYWQdiK811DM4ZvHsJUhERiUZoCcLdxwIlXTa0H/Cqu29390XAfKB3WLGJiMj+RdEHcaOZzQiaoOoEZU2AH4odszQoExGRiJR1ghgCHAp0A1YADwXl8Z5djzv+1swGmVmumeUWFBSEE6WIiJRtgnD3Ve5e5O67gGH8pxlpKdCs2KFNgeV7ucZQd89x95zMzMxwAxYRqcTKdDZXM2vk7iuCt/2B3SOc3gFeNrOHgcZAa2DS/q43ZcqU1Wa2DlgfZ3dGnPI9y4q/rw+sLsn3+BnixVTa5+7vuL3tL2n5/uq1PNfjgZyneiyd81SPpXNeaddj6xJ9qruH8gJeIdaMtJPYHcKVwIvATGAGsaTQqNjx9wALgO+AXxzA5wwtafmeZcXfA7lh1cX+Yi3Nc/d33IHU1/7qLNHq8UDOUz2qHitTPe7tFdodhLtfGKd4+D6Ovxe49yA+6t0DKN+zbG/nhuXnfF5Jz93fcQdSX/HKS1KvYTvYzzuQ81SPpXOe6rF0zgu7HuNK6LmYSpOZ5XoJ5iaRfVM9lg7VY+lQPf48mmrjP4ZGHUAFoXosHarH0qF6/Bl0ByEiInHpDkJEROJSghARkbiUIEREJC4liBIysxpmNsXMzog6lkRlZu3N7CkzG2Vm10UdT6Iys7PMbJiZvW1mJ0cdT6Iys5ZmNtzMRkUdS3lV4RNEvHUpgvJTg7Un5pvZXSW41G+B18KJsvwrjXp09zx3vxYYAFTKoYelVI9vufvVwGXA+SGGW26VUj0udPcrw400sVX4UUxmdjSwCXjB3TsFZcnAXOAkYk95TwYuBJKB+/a4xBVAF2KP7KcBq919dNlEX36URj26e76ZnQncBTzh7i+XVfzlRWnVY3DeQ8BL7j61jMIvN0q5Hke5+7llFXsiKdO5mKLg7mPNLHuP4t7AfHdfCGBmrwL93P0+4CdNSGZ2HFAD6ABsNbMxHptwsNIojXoMrvMO8I6ZvQdUugRRSv8eDbgfeL8yJgcovX+Psm8VPkHsRbz1Jw7b28Hufg+AmV1G7A6iUiWHfTigejSzY4GzgarAmFAjSywHVI/ATcCJQIaZtXL3p8IMLoEc6L/HesSm9+luZncHiUSKqawJosTrT/zXAe4jSj+UhHZA9ejuXwBfhBVMAjvQenwMeCy8cBLWgdbjGuDa8MJJfBW+k3ovSrz+hOyT6rF0qB5Lh+qxlFXWBDEZaG1mLcysCnABsenH5cCoHkuH6rF0qB5LWYVPEGb2CjABaGtmS83sSncvBG4EPgTygNfcfXaUcZZ3qsfSoXosHarHslHhh7mKiMjBqfB3ECIicnCUIEREJC4lCBERiUsJQkRE4lKCEBGRuJQgREQkLiUIKVNmtqkMPuPMEk7hXpqfeayZHX4Q53U3s2eC7cvM7InSj+7AmVn2nlNpxzkm08w+KKuYpOwpQUhCCqZ2jsvd33H3+0P4zH3NXXYscMAJAvgd8PhBBRQxdy8AVpjZEVHHIuFQgpDImNkdZjbZzGaY2Z+Klb8VrN4328wGFSvfZGZ/NrOvgb5mttjM/mRmU81sppm1C47791/iZjbCzB4zs6/MbKGZnRuUJ5nZ4OAzRpvZmN379ojxCzP7XzP7ErjZzH5pZl+b2Tdm9omZNQimnb4WuNXMppnZUcFf168H329yvF+iZpYOdHH36XH2HWJmnwZ186mZNQ/KDzWzicE1/xzvjsxiqx++Z2bTzWyWmZ0flPcK6mG6mU0ys/TgTmFcUIdT490FmVmymT1Q7L/VNcV2vwUMjPsfWBKfu+ulV5m9gE3Bz5OBocRm4EwCRgNHB/vqBj+rAbOAesF7BwYUu9Zi4KZg+3rgmWD7MmILEgGMAP4ZfEYHYusFAJxLbMrxJKAh8CNwbpx4vwAGF3tfh//MQHAV8FCw/UfgN8WOexk4MthuDuTFufZxwOvF3heP+13g0mD7CuCtYHs0cGGwfe3u+tzjuucAw4q9zwCqAAuBXkFZLWKzOVcH0oKy1kBusJ0NzAq2BwH/L9iuCuQCLYL3TYCZUf+70iucV2Wd7luid3Lw+iZ4X5PYL6ixwK/NrH9Q3iwoXwMUAa/vcZ03gp9TiK01Ec9bHlvD41szaxCUHQn8MyhfaWaf7yPWkcW2mwIjzawRsV+6i/ZyzolAB7N/z0Bdy8zS3X1jsWMaAQV7Ob9vse/zIvC3YuVnBdsvAw/GOXcm8KCZ/R8w2t3HmVlnYIW7TwZw9w0Qu9sAnjCzbsTqt02c650MdCl2h5VB7L/JIiAfaLyX7yAJTglComLAfe7+9H8VxhYVOhHo6+5bzOwLYku9Amxz96I9rrM9+FnE3v89by+2bXv8LInNxbYfBx5293eCWP+4l3OSiH2Hrfu47lb+8932p8STprn7XDPrCZwG3GdmHxFrCop3jVuBVUDXIOZtcY4xYndqH8bZl0bse0gFpD4IicqHwBVmVhPAzJqYWRaxv05/DJJDO6BPSJ8/Hjgn6ItoQKyTuSQygGXB9qXFyjcC6cXef0RsZlEAgr/Q95QHtNrL53xFbLpqiLXxjw+2JxJrQqLY/v9iZo2BLe7+D2J3GD2AOUBjM+sVHJMedLpnELuz2AVcTGz95j19CFxnZqnBuW2COw+I3XHsc7STJC4lCImEu39ErIlkgpnNBEYR+wX7AZBiZjOAvxD7hRiG14ktMDMLeBr4GlhfgvP+CPzTzMYBq4uVvwv0391JDfwayAk6db8lzspl7j6H2LKh6XvuC86/PKiHi4Gbg/JbgNvMbBKxJqp4MXcGJpnZNOAe4K/uvgM4H3jczKYDHxP7638wcKmZTST2y35znOs9A3wLTA2Gvj7Nf+7WjgPei3OOVACa7lsqLTOr6e6bLLY28STgCHdfWcYx3ApsdPdnSnh8dWCru7uZXUCsw7pfqEHuO56xQD93/zGqGCQ86oOQymy0mdUm1tn8l7JODoEhwHkHcHxPYp3KBqwjNsIpEmaWSaw/RsmhgtIdhIiIxKU+CBERiUsJQkRE4lKCEBGRuJQgREQkLiUIERGJSwlCRETi+v+jBfEmqzf8qwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Even if the loss is way bigger at the beginning, it doesn't seem to change the learning rate, so let's try to fit."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "20cd1713e7834b3d994ea54840b0eae1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  6%|████▋                                                                    | 4/63 [00:03<00:52,  1.12it/s, loss=326]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-14:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\site-packages\\tqdm\\_tqdm.py\", line 144, in run\n",
      "    for instance in self.tqdm_cls._instances:\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\_weakrefset.py\", line 60, in __iter__\n",
      "    for itemref in self.data:\n",
      "RuntimeError: Set changed size during iteration\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      40.370653  21.679268 \n",
      "    1      24.413239  18.400091                                                                                        \n",
      "    2      19.805302  17.092875                                                                                        \n",
      "    3      17.59664   16.168324                                                                                        \n",
      "    4      16.291103  15.369914                                                                                        \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[15.369914]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lr, 1, cycle_len=5, use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('bias0')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The loss is indeed worse. Now let's even try to do the opposite and initialize the bias at +3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_reg4 = SSD_Head(k,3)\n",
    "models = ConvnetBuilder(f_model, 0, 0, 0, custom_head=head_reg4)\n",
    "learn = ConvLearner(md, models)\n",
    "learn.opt_fn = optim.Adam\n",
    "learn.crit = ssd_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1930ab14d66349249d9202370835fad8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 92%|█████████████████████████████████████████████████████████████▋     | 58/63 [00:31<00:02,  1.87it/s, loss=2.84e+03]"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEOCAYAAACetPCkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd8VFX+//HXJ42QnkBooVdFpAYQFcuqiKu7WFFsKLjYdXV11XV/29z9qruurrp2RLBhF7Ghrq6CKCX0qnQIoQQCCQklkJzfH3N1I4SQgUxuMvN+Ph7zyJ1zz0w+Oca8uffcOdecc4iIiFRXlN8FiIhI/aLgEBGRoCg4REQkKAoOEREJioJDRESCouAQEZGgKDhERCQoCg4REQmKgkNERIKi4BARkaDE+F1AKDRu3Ni1bdvW7zJEROqVWbNmbXHOZR6qX1gGR9u2bcnJyfG7DBGResXM1lSnn05ViYhIUBQcIiISFAWHiIgERcEhIiJBUXCIiEhQFBwiIhKUsLwc93DtKyvn08WbyEiMo1FiHBmJcaQlxBEdZX6XJiJSZyg4KijYWcoNr8z+SVuUQXpCIEQyEuPITG5A05R4mqYEvjZJ/t92YgMNp4iEP/2lqyA9IY6Pbx1IQUkpW4r3UFBSSkFJKVtLSikoDmwvzivii6Wb2VladsDrkxvEkJXekJbpCbRMb1jhkUCr9ARSE2J9+KlERGqWgqOC2Ogojm6ecsh+zjmK9+xjU9EeNu/YzeaiPWwq2k3e9l2s376L3G07+XbFFkr2C5dGiXF0aJJEh8wkOmQm0tHbzkprSJROh4lIPaHgOAxmRnJ8LMnxsXRsklRpH+cchbv2krstECRrC3ayYnMJK/KL+XjhBrbv3Ptj3/jYKDo3TeaoZskc3Twl8GiWoiMUEamTFBwhYmakJQQm17tlpR6wv6CklBX5xazYXMyyzcV8t3EHny/ZzBs5uT/2aZEaz1HNUzg2K5Xstun0ap1OkuZRRMRn+ivkk8BkewZ922b82OacI794D0s27GDJhiKWbihi8YYi/vvdZpwLTNQf1SyF7Lbp9GkTeGSlNcRMp7lEpPaYc87vGmpcdna2C6fVcXfs3suctdvJWbONWWsKmLN2+4+T81lpDTmxY2MGdm7MCR0ak54Y53O1IlJfmdks51z2IfspOOqffWXlLN24g5zVBUxbWcDUFVvYsXsfZnBsVioDOzXmxI6Z9GmTTlyMPuMpItWj4Ajj4NjfvrJy5q8vZMr3W/h6eT6z126nrNyREBfNqV2acGa3ZpzaJZPkeE22i8jBKTgiKDj2t2P3XqatLOC/323m00Wb2FK8h7joKE7s1Jgzj2nK6Uc3pVFSA7/LFJE6RsERwcFRUVm5Y87abUxauJFJizaSu20XUQb92mUwpGcWZ3dvToqOREQEBYeCoxLOORblFfHJoo18tGADK/JLiI+NYvAxzbiwTysGdGikdblEIpiCQ8FRJecc83ILeWvWOibOzaNo9z5apMZzfu+WXNCnJe0aJ/pdoojUMgWHgqPadu8t4z9LNvHWrFwmf59PuYMB7RtxzcB2nNqliZZDEYkQCg4Fx2HZVLSbt2fn8tK3a9hQuJv2jRMZcWI7LujdkoZx0X6XJyIhpOBQcByRvWXlfLxwI6OnrGR+biHpCbFcflwbrhjQhibJ8X6XJyIhoOBQcNQI5xwzV29j9JSVfLZkE7FRUZzbqwW3nNaJlukJfpcnIjWousGhtaqkSmZGv3YZ9GuXweotJYyZuorXZq7j3Tnruax/G248tSOZyfpMiEgk0RGHBC1v+y4e/2IZb+TkEhcdxYgT2zLqpA6kNtTnQUTqM52qUnCE3KotJTz82fe8Py+PlPgYrjulA1cd35aEOB3IitRH1Q0OrYAnh61d40QeH9aLD285key2Gfx90nec/I8veX9eHuH4DxIRCVBwyBE7pkUqY67qy9vXD6BZSjw3j5/DyHE5rN++y+/SRCQEFBxSY/q0yeDdG47n92cfzbcrtjLo4a94Yeoqysp19CESThQcUqNioqO4ZmB7Pr3tJLLbZvDn9xdz/lPfsHRjkd+liUgNUXBISLTKSGDs1X159JKe5Bbs5JzHvubvk5aye2+Z36WJyBFScEjImBlDembxn9tPZkjPLJ78cgUXPPUNa7aW+F2aiBwBBYeEXHpiHP8c2oPRV2aTu20X5zz2NR8v2OB3WSJymEIWHGY2xsw2m9nCSvbdYWbOzBp7z83MHjOz5WY238x6V+g73MyWeY/hoapXQu/0rk358JYTad8kietfmc2fJi5izz6duhKpb0J5xDEWGLx/o5m1As4A1lZoPgvo5D1GAU95fTOAPwL9gX7AH80sPYQ1S4i1TE/gzWsHMOKEdoz9ZjVDn/6WdQU7/S5LRIIQsuBwzk0GCirZ9QjwW6DiNZpDgBddwDQgzcyaA2cCnznnCpxz24DPqCSMpH6Ji4niD7/oytOX92bllhLOfmwKny7a6HdZIlJNtTrHYWa/BNY75+bttysLWFfhea7XdrB2CQODuzXnw5sH0qZRIqNemsXfJy2lXJ/5EKnzai04zCwBuBf4Q2W7K2lzVbRX9v6jzCzHzHLy8/MPv1CpVa0bJfDW9QMY1q8VT365gtvfmEvpvnK/yxKRKtTmEUcHoB0wz8xWAy2B2WbWjMCRRKsKfVsCeVW0H8A596xzLts5l52ZmRmC8iVUGsRE83/nHcudZ3Zhwtw8rh47g6Lde/0uS0QOotaCwzm3wDnXxDnX1jnXlkAo9HbObQQmAld6V1cdBxQ65zYAnwCDzCzdmxQf5LVJmDEzbjy1Iw9d1IPpKwsY+vS3bCra7XdZIlKJUF6OOx74FuhiZrlmNrKK7h8BK4HlwHPADQDOuQLgPmCm9/iL1yZh6sI+LRlzVV/WFezkvCemsmzTDr9LEpH96H4cUictXF/I1WNnsmdvGc9dmU3/9o38Lkkk7Ol+HFKvdctK5Z3rj6dxcgOueH4GH+mT5iJ1hoJD6qxWGQm8fd3xdG+Zys3j5zBpoT7rIVIXKDikTktPjGPciH50b5nKLePnMPl7XWot4jcFh9R5iQ1iGHtVPzo0SWLUSznMXK3rI0T8pOCQeiE1IZYXR/SjRWpDRrwwk4XrC/0uSSRiKTik3shMbsDL1/QnpWEsV46ZwfLNulRXxA8KDqlXWqQ15OVr+hNlxmWjp2tlXREfKDik3mnXOJGXr+nH7r3lXDp6GhsL9Qlzkdqk4JB66ahmKYwb0Y+C4lIuf346hTu1tpVIbVFwSL3Vs1Uao4f3Zc3WEq5/ZRZ7y7SqrkhtUHBIvTagQyPuP78736zYyh/eW0Q4LqEjUtfE+F2AyJG6sE9LVuYX8+SXK+iQmcg1A9v7XZJIWFNwSFi4Y1AXVuaX8LePltCucSKnHd3U75JEwpZOVUlYiIoyHr64B8e0SOGW8XNYsqHI75JEwpaCQ8JGQlwMo6/sS1J8DNeMyyF/xx6/SxIJSwoOCSvNUuN5fnhfCkpKGfVSDrv3lvldkkjYUXBI2OmWlcojF/dkztrt3PnWfF1pJVLDFBwSlgZ3a8Zdg4/i/Xl5PDN5pd/liIQVBYeEretObs/ZxzbnH598x6w1WopdpKYoOCRsmRn3X3AsLdLiuWX8XLbvLPW7JJGwoOCQsJYSH8sTl/Zm847d3PGm5jtEaoKCQ8Je95Zp3HPW0fxnySZemLra73JE6j0Fh0SEq09oy+lHN+X+j5cwb912v8sRqdcUHBIRzIyHLupOk+R4bho/m6LdWoZd5HApOCRipCXE8diwXuRt383db2u+Q+RwKTgkovRpk86dZ3bhowUbeXn6Wr/LEamXFBwScUYNbM/JnTO574PFLMor9LsckXpHwSERJyrKeHhoD9ITYrn1tblaz0okSAoOiUiNkhrw0EU9WL65mL9P+s7vckTqFQWHRKyBnTIZPqANY6auYuryLX6XI1JvKDgkot191tG0z0zkjjfnUbhLl+iKVIeCQyJaw7hoHhnak8079vDH9xb6XY5IvaDgkIjXo1UaN/+sIxPm5vHB/Dy/yxGp8xQcIsCNp3akR6s07n13IRsLd/tdjkidFrLgMLMxZrbZzBZWaPuHmS01s/lm9q6ZpVXYd4+ZLTez78zszArtg7225WZ2d6jqlcgWGx3FI0N7sGdfGb/Vp8pFqhTKI46xwOD92j4DujnnugPfA/cAmFlX4BLgGO81T5pZtJlFA08AZwFdgWFeX5Ea1z4ziXt/fjSTv8/n5Wlr/C5HpM4KWXA45yYDBfu1feqc2+c9nQa09LaHAK855/Y451YBy4F+3mO5c26lc64UeM3rKxISlx/XhpM6Z/K3j5awIr/Y73JE6iQ/5zhGAB9721nAugr7cr22g7WLhISZ8Y8Lu9MgJprbX5/L3rJyv0sSqXN8CQ4zuxfYB7zyQ1Ml3VwV7ZW95ygzyzGznPz8/JopVCJS05R4/nZeN+blFvLEf5f7XY5InVPrwWFmw4FzgMvc/2Ygc4FWFbq1BPKqaD+Ac+5Z51y2cy47MzOz5guXiHJO9xac27MFj3+xXDd+EtlPrQaHmQ0G7gJ+6ZzbWWHXROASM2tgZu2ATsAMYCbQyczamVkcgQn0ibVZs0SuPw/pRpPkBtz2+lx2lWohRJEfhPJy3PHAt0AXM8s1s5HAv4Fk4DMzm2tmTwM45xYBbwCLgUnAjc65Mm8i/SbgE2AJ8IbXVyTkUhvG8s+LerBySwn3f7zE73JE6gwLx+vVs7OzXU5Ojt9lSJj4y/uLGTN1FeNG9OPkzjoNKuHLzGY557IP1U+fHBc5hN8O7kKnJknc+eY8tpWU+l2OiO8UHCKHEB8bzSMX92TbzlJ+/95CfapcIp6CQ6QaumWl8uvTO/Ph/A1MnKeFECWyKThEqunak9rTp006v5+wkLztu/wuR8Q3Cg6RaoqJjuLhoT0oK3fc+dY8yst1ykoik4JDJAhtGiXyh3O6MnX5VsZMXeV3OSK+UHCIBOnivq0Y1LUpf5/0HYvyCv0uR6TWKThEgmRmPHBBd9ISYrll/Bx9qlwijoJD5DBkJMbx8NCerMgv4W8fLfa7HJFapeAQOUwndmrMqJPa8/K0tXy2eJPf5YjUGgWHyBH4zaDOdG2ewl1vz2dzke5VLpFBwSFyBBrERPPYsF7sLN3Hb97UJboSGRQcIkeoY5Mk/t85XZmybIsu0ZWIUK3gMLNbzSzFAp43s9lmNijUxYnUF5f2a80Z3iW6i/OK/C5HJKSqe8QxwjlXBAwCMoGrgQdCVpVIPWNmPOhdonvra7pEV8JbdYPjh3t//xx4wTk3j8rvBy4SsTIS4/jn0B4s21zMXz7QJboSvqobHLPM7FMCwfGJmSUD5aErS6R+Gtgpk+tP6cD4GWt5b+56v8sRCYnqBsdI4G6gr3ev8FgCp6tEZD+/OaMz2W3S+d07C1iZX+x3OSI1rrrBMQD4zjm33cwuB34PaJEekUrEREfx+KW9iIuJ4sZX57B7r+Y7JLxUNzieAnaaWQ/gt8Aa4MWQVSVSzzVPbcjDF/dkyYYizXdI2KlucOxzgftlDgEedc49CiSHriyR+u/ULk247uQOvDpd8x0SXqobHDvM7B7gCuBDM4smMM8hIlX4zSDNd0j4qW5wXAzsIfB5jo1AFvCPkFUlEiZio6N4bJjmOyS8VCs4vLB4BUg1s3OA3c45zXGIVEOLtIY8PDQw33Gf5jskDFR3yZGhwAzgImAoMN3MLgxlYSLh5NSjmnDtye15RfMdEgZiqtnvXgKf4dgMYGaZwH+At0JVmEi4uWNQF2at3sbdby+gU5NkurZI8bskkcNS3TmOqB9Cw7M1iNeKCIH5jicv701KwxhGvZTDtpJSv0sSOSzV/eM/ycw+MbOrzOwq4EPgo9CVJRKemiTH88wV2Wwu2sONr85mX5lW7pH6p7qT43cCzwLdgR7As865u0JZmEi46tkqjb+d141vVmzl/o+X+l2OSNCqO8eBc+5t4O0Q1iISMS7KbsWivCKe/3oVx7RI4fzeLf0uSaTaqgwOM9sBVHYvTAOcc06zeyKH6d6zj2bpxiLufmcBHZsk0b1lmt8liVRLlaeqnHPJzrmUSh7JCg2RIxMbHcUTl/YmM6kB1740i/wde/wuSaRadGWUiI8aJTXgmSv6sG1nKTe+Mpu9miyXekDBIeKzblmpPHhBd2asLuDP7y8isJ6oSN0VsuAwszFmttnMFlZoyzCzz8xsmfc13Ws3M3vMzJab2Xwz613hNcO9/svMbHio6hXx05CeWVx7UntenraWMVNX+12OSJVCecQxFhi8X9vdwOfOuU7A595zgLOATt5jFIH7f2BmGcAfgf5AP+CPP4SNSLi5a/BRDD6mGX/9cDGfLNrodzkiBxWy4HDOTQYK9mseAozztscB51Zof9EFTAPSzKw5cCbwmXOuwDm3DfiMA8NIJCxERRmPXNyT7i3TuPW1Ocxbt93vkkQqVdtzHE2dcxsAvK9NvPYsYF2Ffrle28HaRcJSw7hoRl+ZTeOkBowcl8O6gp1+lyRygLoyOW6VtLkq2g98A7NRZpZjZjn5+fk1WpxIbcpMbsALV/Vlz74yRoydSeGuvX6XJPITtR0cm7xTUHhff1g4MRdoVaFfSyCvivYDOOeedc5lO+eyMzMza7xwkdrUqWkyz1zeh1VbSrjhlVmU7tNlulJ31HZwTAR+uDJqOPBehfYrvaurjgMKvVNZnwCDzCzdmxQf5LWJhL3jOzbmgQu6M3X5Vu59d4Eu05U6o9prVQXLzMYDpwCNzSyXwNVRDwBvmNlIYC2BG0NBYKXdnwPLgZ3A1QDOuQIzuw+Y6fX7i3Nu/wl3kbB1YZ+WrN1awmNfLKdNowRu+lknv0sSCV1wOOeGHWTXaZX0dcCNB3mfMcCYGixNpF657YzOrC3YyUOffk/jpAZc0q+13yVJhAtZcIhIzTAzHrywOwU793LPuwtIbBDDL3q08LssiWB15aoqEalCg5honrm8D33bZHDb63P5Yukmv0uSCKbgEKknGsZFM/qqbI5qnsz1L8/m2xVb/S5JIpSCQ6QeSYmP5cUR/WmdkcA142YyV58uFx8oOETqmYzEOF6+pj8ZSXEMHzODpRuL/C5JIoyCQ6QeapoSzysjjyM+NorLR89g9ZYSv0uSCKLgEKmnWjdK4OWR/Sl3jstGTydv+y6/S5IIoeAQqcc6NU3mxRH9KNq1l2HPTWNDocJDQk/BIVLPdctK5cWR/dhaXMqlz01nU9Fuv0uSMKfgEAkDvVqnM25EXzYX7WbYc9PYvEPhIaGj4BAJE33aZDB2RD82Fu7m0uemk79jj98lSZhScIiEkb5tMxhzVV/Wb9vFZaOnsbVY4SE1T8EhEmaOa9+I56/KZm3BTi4bPZ2CklK/S5Iwo+AQCUPHd2jM6Cv7smpLCZePns72nQoPqTkKDpEwdWKnxjx7ZTbL84sZ9tx0tui0ldQQBYdIGDu5cybPD89m1ZZihj7zrT7nITVCwSES5gZ2yuTFEf3ZXLSHi57+lrVbd/pdktRzCg6RCNCvXQav/qo/xXv2cdEz37B88w6/S5J6TMEhEiG6t0zj9VEDKHcw9JlpLFxf6HdJUk8pOEQiSJdmybxx7QDiY6IY9tw0Zq0p8LskqYcUHCIRpl3jRN68/ngaJcZxxfMzmLp8i98lST2j4BCJQFlpDXnjugG0Sk/g6hdm8uH8DX6XJPWIgkMkQjVJjuf1a4+jR6tUbho/m7FTV/ldktQTCg6RCJaWEMdLI/tzxtFN+dP7i3lw0lKcc36XJXWcgkMkwsXHRvPU5X24tH9rnvpyBb95cx57y8r9LkvqsBi/CxAR/0VHGX87txvNUuJ5+LPv2VpcypOX9Saxgf5EyIF0xCEiAJgZt5zWiQfOP5Ypy/K59Dktyy6VU3CIyE9c0q81z16RzXebdnDBU9+wMr/Y75KkjlFwiMgBTu/alFeuOY4du/dx7hNT9VkP+QkFh4hUqk+bdCbceALNUxty5ZgZvDxtjd8lSR2h4BCRg2qVkcBb1w/g5M6Z/H7CQv40cRH7dMVVxFNwiEiVkuNjee7KbH41sB1jv1nN1WNnUrhrr99liY8UHCJySNFRxr1nd+XBC47l2xVbOf/JqazeUuJ3WeITX4LDzG4zs0VmttDMxptZvJm1M7PpZrbMzF43szivbwPv+XJvf1s/ahYRuLhva14a2Z+tJaWc++RUpizL97sk8UGtB4eZZQG3ANnOuW5ANHAJ8CDwiHOuE7ANGOm9ZCSwzTnXEXjE6yciPhnQoRETbjiBzKQGXDlmBo9/vozyci1TEkn8OlUVAzQ0sxggAdgA/Ax4y9s/DjjX2x7iPcfbf5qZWS3WKiL7ads4kQk3nsAve7Tgn599z8hxM9m+s9TvsqSW1HpwOOfWAw8BawkERiEwC9junNvndcsFsrztLGCd99p9Xv9GtVmziBwosUEM/7q4J/cNOYavl2/h7Me+ZkGu7ioYCfw4VZVO4CiiHdACSATOqqTrD8e+lR1dHHBcbGajzCzHzHLy83XeVaQ2mBlXDGjLG9cOwDnHBU99w6vT12qF3TDnx6mq04FVzrl859xe4B3geCDNO3UF0BLI87ZzgVYA3v5U4ID7XTrnnnXOZTvnsjMzM0P9M4hIBb1ap/PBLQPp3z6D3727gN+8OY9dpWV+lyUh4kdwrAWOM7MEb67iNGAx8F/gQq/PcOA9b3ui9xxv/xdO/5wRqXMyEuMYe3U/bj2tE+/OWc8v/v01i/J06ioc+THHMZ3AJPdsYIFXw7PAXcDtZracwBzG895Lngcaee23A3fXds0iUj3RUcZtZ3TmpRH9Kdq1l3OfmMozX63QVVdhxsLxH+/Z2dkuJyfH7zJEItq2klLueWcBkxZt5Lj2GTw8tCct0hr6XZZUwcxmOeeyD9VPnxwXkZBIT4zjqct78/cLujM/t5DB/5rM+/PyDv1CqfMUHCISMmbG0L6t+OiWgbTPTOLm8XO4/fW57Nitta7qMwWHiIRc28aJvHndAG45rRMT5q5n8L+mMPl7XTZfXyk4RKRWxEZHcfsZnXnzuuNpEBvFlWNm8Nu35mml3XpIwSEitapPm3Q+umUg15/Sgbdm5TLoka/4fMkmv8uSICg4RKTWxcdGc9fgo5hw4wmkJ8QxclwOv35tDttKtN5VfaDgEBHfdG+ZxsSbTuTW0zrxwfwNnPHIV3y0YIPfZckhKDhExFdxMVHcdkZn3r/5RJqlxnPDK7O59qUcNhXt9rs0OQgFh4jUCUc3T2HCDSdw91lH8eV3+Zz+8FeMn7FWnzqvgxQcIlJnxERHcd3JHZj065M4pkUK97yzgEtHT2OVblNbpyg4RKTOadc4kfG/Oo4Hzj+WRXlFDP7XZJ76cgV7y8r9Lk1QcIhIHWVmXNKvNZ/ffjKndmnCg5OWMuTfU5mzdpvfpUU8BYeI1GlNUuJ5+oo+PH15b7YU7+G8J7/hjjfnkb9jj9+lRSwFh4jUC4O7NeeLO07h2pPb897c9fzsoS8ZPWWlTl/5QMEhIvVGUoMY7jnraCb9+iR6t0nnrx8u4axHp/D1si1+lxZRFBwiUu90yExi7NV9ee7KbEr3lXP589O57qVZrCvY6XdpESHm0F1EROoeM+OMrk0Z2Kkxz01eyRNfLueLpZsZfnwbbjq1E6kJsX6XGLZ0xCEi9Vp8bDQ3n9aJ/95xCr/s2YLRX6/ipH/8l9FTVrJnX5nf5YUlBYeIhIXmqQ156KIefHjzQLq3TOWvHy7h9Ie/4oP5eYTjLbIr882KLcxbtz3k30fBISJhpWuLFF4a2Z8XR/QjMS6Gm16dw3lPfsPU5VvCOkDKyh2/n7CQu99ZEPKfU3McIhKWTuqcyQkdG/PO7Fwe+vQ7Lhs9nY5NkrjiuDac1zuLlPjwmgP5YH4eK/NLeOqy3phZSL+XhWMCZ2dnu5ycHL/LEJE6YvfeMj6Yv4GXvl3NvNxCEuKiObdXFlcc14ajm6f4Xd4RKy93DPrXZKIMJt16ElFRhxccZjbLOZd9qH464hCRsBcfG82FfVpyYZ+WzFu3nZenreHtWbm8On0t2W3SuWJAG848phnxsdF+l3pYPl64keWbi3l8WK/DDo1g6IhDRCLS9p2lvDUrl5enrWH11p2kJcRyXq8shvVrTeemyX6XV23l5Y6fPzaFvWXlfHrbyUQfQXDoiENEpAppCXFcM7A9I05oxzcrtjJ+5lpenraGF6auplfrNIb1bc05PZqTEFe3/0x+ungTSzfu4F8X9zyi0AiGjjhERDwFJaW8MzuX8TPWsiK/hKQGMZzTvTm926TTsUkSHZsk1alJdeccZz/2Nbv2lvHZbScRE31kF8rqiENEJEgZiYGjkJEntmPWmm2Mn7GO9+bm8drMdT/2aZLc4McQ6dgkiWNapHJMixRf5kc+X7KZxRuKeOiiHkccGsHQEYeISBX2lZWzbtsulm8u/t8jv5gVm4sp3rMPgNhoo2vzFHq1TqdnqzR6tU6jdUZCSC+Ldc4x5ImpbN+5l89/czKxNRAcOuIQEakBMdFRtGucSLvGiZzRtemP7c45NhbtZn5uIXPWbmfO2m28kbOOsd+sBgJHLz1aptItK5VjWqTSLSuFrLSGNRYmX36fz/zcQh684NgaCY1gKDhERA6DmdE8tSHNUxty5jHNgMDRyfebipm7LhAkC9YXMnnZFsrKA2d20hJi6ead2uraIoVOTZJpn5kY9Gku5xyP/mcZWWkNOa9Xyxr/2Q5FwSEiUkNioqPo6oXCpf1bA4EPHy7duIOF6wtZlFfIwvVFvDB1NaXeDajMoHVGAh0zk+jYNImOmUl0aZZMtxapB/1MxpRlW5i7bjv/d96xxMXU/spRCg4RkRCKj42mZ6s0erZK+7GtdF85K7cE5kuWbfrf3MnkZfnsLQscnTRLiecXPZozpGcWx7RI+fEUl3OORz9fRovUeC7ok+XLz6TgEBGpZXExURzVLIWjmv10uZN9ZeWsLdjJ/NxCPpifxwtTV/PclFV0yExkSM8shvRswfptu5i1Zhv3DTmGBjH+fNJdV1WJiNRR20pK+XjhRibMXc+z6bn2AAAKAUlEQVSMVQVA4Pa5iQ2i+erOU2v8EuA6fVWVmaUBo4FugANGAN8BrwNtgdXAUOfcNgscnz0K/BzYCVzlnJvtQ9kiIrUqPTGOS/u35tL+rcnbvov35+XxyaKNDD++ra/ravlyxGFm44ApzrnRZhYHJAC/Awqccw+Y2d1AunPuLjP7OXAzgeDoDzzqnOtf1fvriENEJHjVPeKo9el4M0sBTgKeB3DOlTrntgNDgHFet3HAud72EOBFFzANSDOz5rVctoiIePy4A2B7IB94wczmmNloM0sEmjrnNgB4X5t4/bOAdRVen+u1iYiID/wIjhigN/CUc64XUALcXUX/yi5kPuD8mpmNMrMcM8vJz8+vmUpFROQAfgRHLpDrnJvuPX+LQJBs+uEUlPd1c4X+rSq8viWQt/+bOueedc5lO+eyMzMzQ1a8iEikq/XgcM5tBNaZWRev6TRgMTARGO61DQfe87YnAldawHFA4Q+ntEREpPb59QHAm4FXvCuqVgJXEwixN8xsJLAWuMjr+xGBK6qWE7gc9+raL1dERH7gS3A45+YClV3ydVolfR1wY8iLEhGRavFjjkNEROqxsFxyxMzygTVAKlBYSZf926t63hjYEoIyK/u+Nfm6qvoEuy/Sx6uq/Rqv4PYf6XhB6MZM4wVtnHOHvrrIORe2D+DZ6rRX9RzIqe36auJ1VfUJdl+kj1dV+zVetTteoRwzjVf1H+F+qur9arYf6nmoHO73qc7rquoT7L5IH6+q9mu8gtuv8Qpuf50cr7A8VVWTzCzHVWPtFgnQeAVH4xU8jVlwQjFe4X7EUROe9buAekbjFRyNV/A0ZsGp8fHSEYeIiARFRxwiIhIUBYeIiARFwSEiIkFRcBwhM0s0s1lmdo7ftdR1Zna0mT1tZm+Z2fV+11PXmdm5Zvacmb1nZoP8rqeuM7P2Zva8mb3ldy11lff3apz3e3XZ4b5PxAaHmY0xs81mtnC/9sFm9p2ZLfduYXsodwFvhKbKuqMmxss5t8Q5dx0wlMrXKgsbNTReE5xzvwKuAi4OYbm+q6HxWumcGxnaSuueIMfufOAt7/fql4f7PSM2OICxwOCKDWYWDTwBnAV0BYaZWVczO9bMPtjv0cTMTiewJPym2i7eB2M5wvHyXvNL4Gvg89otv9aNpQbGy/N773XhbCw1N16RZizVHDsC9zP64Y6qZYf7Df1aVt13zrnJZtZ2v+Z+wHLn3EoAM3sNGOKcux844FSUmZ0KJBL4D7PLzD5yzpWHtHCf1MR4ee8zEZhoZh8Cr4auYn/V0O+XAQ8AHzvnZoe2Yn/V1O9XJApm7AjcGK8lMJcjOHCI2OA4iMrub97/YJ2dc/cCmNlVwJZwDY0qBDVeZnYKgUPlBgTusxJpghovAvetOR1INbOOzrmnQ1lcHRTs71cj4G9ALzO7xwuYSHWwsXsM+LeZnc0RLE2i4Pipat3f/IAOzo2t+VLqhaDGyzn3JfBlqIqpB4Idr8cI/I8eqYIdr63AdaErp16pdOyccyXUwM3wInmOozLVur+5/EjjFRyNV3A0XocvpGOn4PipmUAnM2vn3db2EgL3PJfKabyCo/EKjsbr8IV07CI2OMxsPPAt0MXMcs1spHNuH3AT8AmwBHjDObfIzzrrCo1XcDRewdF4HT4/xk6LHIqISFAi9ohDREQOj4JDRESCouAQEZGgKDhERCQoCg4REQmKgkNERIKi4JA6wcyKa+F7/LKaS+XX5Pc8xcyOP4zX9TKz0d72VWb275qvLnhm1nb/5bsr6ZNpZpNqqyapfQoOCSvectKVcs5NdM49EILvWdWab6cAQQcH8Dvg8cMqyGfOuXxgg5md4HctEhoKDqlzzOxOM5tpZvPN7M8V2idY4G6Li8xsVIX2YjP7i5lNBwaY2Woz+7OZzTazBWZ2lNfvx3+5m9lYM3vMzL4xs5VmdqHXHmVmT3rf4wMz++iHffvV+KWZ/Z+ZfQXcama/MLPpZjbHzP5jZk29pa6vA24zs7lmNtD71/jb3s83s7I/rmaWDHR3zs2rZF8bM/vcG5vPzay1197BzKZ57/mXyo7gLHD3tw/NbJ6ZLTSzi732vt44zDOzGWaW7B1ZTPHGcHZlR01mFm1m/6jw3+raCrsnAId9hzmp45xzeujh+wMo9r4OAp4lsLpnFPABcJK3L8P72hBYCDTynjtgaIX3Wg3c7G3fAIz2tq8C/u1tjwXe9L5HVwL3LgC4kMCS71FAM2AbcGEl9X4JPFnheTr/W4nhGuCf3vafgDsq9HsVONHbbg0sqeS9TwXervC8Yt3vA8O97RHABG/7A2CYt33dD+O53/teADxX4XkqEAesBPp6bSkEVs1OAOK9tk5AjrfdFljobY8Cfu9tNwBygHbe8yxggd+/V3qE5qFl1aWuGeQ95njPkwj84ZoM3GJm53ntrbz2rQTuZPb2fu/zjvd1FoF7gFRmggvcQ2WxmTX12k4E3vTaN5rZf6uo9fUK2y2B182sOYE/xqsO8prTga5mP656nWJmyc65HRX6NAfyD/L6ARV+npeAv1doP9fbfhV4qJLXLgAeMrMHgQ+cc1PM7Fhgg3NuJoBzrggCRycE7tvQk8D4dq7k/QYB3SsckaUS+G+yCtgMtDjIzyD1nIJD6hoD7nfOPfOTxsBNoE4HBjjndprZl0C8t3u3c27/22Du8b6WcfDf8z0Vtm2/r9VRUmH7ceBh59xEr9Y/HeQ1UQR+hl1VvO8u/vezHUq1F5tzzn1vZn2AnwP3m9mnBE4pVfYetxG4JXIPr+bdlfQxAkd2n1SyL57AzyFhSHMcUtd8AowwsyQAM8uywP2kU4FtXmgcBRwXou//NXCBN9fRlMDkdnWkAuu97eEV2ncAyRWef0pg1VIAvH/R728J0PEg3+cbAktkQ2AO4WtvexqBU1FU2P8TZtYC2Omce5nAEUlvYCnQwsz6en2Svcn+VAJHIuXAFUBlFx18AlxvZrHeazt7RyoQOEKp8uorqb8UHFKnOOc+JXCq5VszWwC8ReAP7yQgxszmA/cR+EMZCm8TuAnOQuAZYDpQWI3X/Ql408ymAFsqtL8PnPfD5DhwC5DtTSYvppI71jnnlhK4XWzy/vu811/tjcMVwK1e+6+B281sBoFTXZXVfCwww8zmAvcCf3XOlQIXA4+b2TzgMwJHC08Cw81sGoEQKKnk/UYDi4HZ3iW6z/C/o7tTgQ8reY2EAS2rLrIfM0tyzhVb4B7WM4ATnHMba7mG24AdzrnR1eyfAOxyzjkzu4TARPmQkBZZdT2TgSHOuW1+1SChozkOkQN9YGZpBCa576vt0PA8BVwURP8+BCazDdhO4IorX5hZJoH5HoVGmNIRh4iIBEVzHCIiEhQFh4iIBEXBISIiQVFwiIhIUBQcIiISFAWHiIgE5f8DeQJuIhrTeT4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It looks like this time we can pick a higher learning rate. The loss at the beginning is also way higher than before!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28e25877f42a4ecdb282ebbac7f65bcd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                           | 0/63 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-20:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\site-packages\\tqdm\\_tqdm.py\", line 144, in run\n",
      "    for instance in self.tqdm_cls._instances:\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\_weakrefset.py\", line 60, in __iter__\n",
      "    for itemref in self.data:\n",
      "RuntimeError: Set changed size during iteration\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      87.238823  26.501392 \n",
      "    1      36.56298   19.266548                                                                                        \n",
      "    2      24.075832  31.767622                                                                                        \n",
      "    3      20.066492  31.117821                                                                                        \n",
      "    4      18.413494  16.906162                                                                                        \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[16.906162]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(5*lr, 1, cycle_len=5, use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('bias3')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Again, the loss is worse than before. There's really an interest in initializing the bias woith a negative value. Now let's see what those bias look like when the training is finished."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('bias-3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "outConv1 = learn.model[8].out.oconv1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "-3.0918\n",
       "[torch.cuda.FloatTensor of size 1 (GPU 0)]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outConv1.bias.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('bias0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "1.00000e-02 *\n",
       " -6.3792\n",
       "[torch.cuda.FloatTensor of size 1 (GPU 0)]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outConv1 = learn.model[8].out.oconv1\n",
    "outConv1.bias.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('bias3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 2.7783\n",
       "[torch.cuda.FloatTensor of size 1 (GPU 0)]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outConv1 = learn.model[8].out.oconv1\n",
    "outConv1.bias.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is very surprising (for me). I would have thought that the model would have learned to put a negative bias, since it seems to make it easier for him, but he didn't do that. The bias have barely changed from their initial values. \n",
    "\n",
    "Now let's see if we can manage the same error without bias as when we initialized with -3, just by training longer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('bias0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "698886411aec4792b47076a38ad2d1f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      18.60108   17.518761 \n",
      "    1      17.316723  16.283512                                                                                        \n",
      "    2      16.029754  17.229944                                                                                        \n",
      "    3      14.924728  15.018361                                                                                        \n",
      "    4      14.035353  14.558006                                                                                        \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[14.558006]"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lr, 1, cycle_len=5, use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Even with a second cycle we're still not as good as the model with bias initialized at -3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_reg4 = SSD_Head(k,0)\n",
    "models = ConvnetBuilder(f_model, 0, 0, 0, custom_head=head_reg4)\n",
    "learn = ConvLearner(md, models)\n",
    "learn.opt_fn = optim.Adam\n",
    "learn.crit = ssd_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e38f4f20b4c1402387b0eb459c83bf1c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      42.762562  21.587082 \n",
      "    1      24.99543   19.196276                                                                                        \n",
      "    2      20.081579  17.913952                                                                                        \n",
      "    3      18.114532  16.890486                                                                                        \n",
      "    4      16.93616   16.752743                                                                                        \n",
      "    5      15.928686  16.028091                                                                                        \n",
      "    6      15.101899  15.778208                                                                                        \n",
      "    7      14.549666  16.048922                                                                                        \n",
      "    8      13.896238  15.096319                                                                                        \n",
      "    9      13.331165  15.014502                                                                                        \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[15.014502]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lr, 1, cycle_len=10, use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Doing a longer cycle doesn't help either."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What about even stronger negative bias?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_reg4 = SSD_Head(k,-6)\n",
    "models = ConvnetBuilder(f_model, 0, 0, 0, custom_head=head_reg4)\n",
    "learn = ConvLearner(md, models)\n",
    "learn.opt_fn = optim.Adam\n",
    "learn.crit = ssd_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b85c262c75f48b590c225352ee187a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 79%|█████████████████████████████████████████████████████████▏              | 50/63 [00:27<00:07,  1.83it/s, loss=181]"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEOCAYAAACaQSCZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4VOXZx/HvnX2DsIUlIRD2RVYJm+ACKiIK7rtURYu2aMW6VGtftdpWq9alLq0UUbGioiKioIgKIgKBBNnDGrZAgLAHQvb7/WNGG3ECE8jMmZncn+uaK2fOnGfmniPml+c85zxHVBVjjDHmRMKcLsAYY0xwsMAwxhjjFQsMY4wxXrHAMMYY4xULDGOMMV6xwDDGGOMVCwxjjDFescAwxhjjFQsMY4wxXrHAMMYY45UIpwuoSY0aNdK0tDSnyzDGmKCRlZW1R1WTvNk2pAIjLS2NzMxMp8swxpigISJbvN3WDkkZY4zxigWGMcYYr1hgGGOM8YoFhjHGGK9YYBhjjPGKzwJDRFJFZLaIZIvIKhG5u9Jrd4nIWvf6p6tov1lEVojIUhGxU5+MMcZhvjyttgy4V1WXiEgdIEtEZgFNgEuAbqpaLCKNj/Meg1R1jw9rRFVZmLOPpDpRtG1cx5cfZYwxQc1nPQxVzVPVJe7lAiAbSAF+AzylqsXu13b7qgZvFJaUc9tbi/n3tzlOlmGMMQHPL2MYIpIG9AQygPbAmSKSISLfikjvKpop8KWIZInIaF/VFh8dwYgeKXy2fAcHj5b66mOMMSbo+TwwRCQB+AgYq6qHcB0Gqw/0A+4HJouIeGg6QFVPBy4ExojIWVW8/2gRyRSRzPz8/JOq8Ya+LSgqrWDqD9tPqr0xxtQGPg0MEYnEFRbvqOoU9+pcYIq6LAIqgEbHtlXVHe6fu4GPgT6ePkNVx6lquqqmJyV5NR3KL3RJSaRrSiKTMraiqif1HsYYE+p8eZaUAK8D2ar6XKWXpgKD3du0B6KAPce0jXcPlCMi8cAQYKWvagW4vm8L1u4qYMnWA778GGOMCVq+7GEMAEYCg92nxi4VkWHABKC1iKwE3gNuUlUVkWQRmeFu2wSYJyLLgEXAdFX9woe1Mrx7MvFR4UzK2OrLjzHGmKDls9NqVXUe4GlsAuBGD9vvAIa5l3OA7r6qzZOE6Agu6ZnCR1m5PHJxZxLjIv358cYYE/DsSu9Kru/TguKyCj7+IdfpUowxJuBYYFTSJSWRbs0TeXfRNhv8NsaYY1hgHOO6Pj8Ofu93uhRjjAkoFhjH+N/g9zanSzHGmIBigXGMHwe/P1u+g4OFNXvlt6qyPPcApeUVNfq+xhjjDxYYHvhi8LukrIKHp65kxMvf8/fP19TY+xpjjL9YYHjw4+D3pEU1c+X3viMljHw9g0kZW2ndKJ63Fmxm054jp16oMcb4kQVGFa7r04J1uw6f8uD3mp2HGPHyPH7YdoAXrunBe7f3Iyo8jL/NyK6hSo0xxj8sMKowwj34/c4pXPn95aqdXPHqfErKKph8e38u7ZlC4zox/HZQW2at3sX8DT691YcxxtQoC4wqxLsHv6cvz6v24Leq8srsDdz+3yzaNE5g2p0D6ZFa76fXbx3YipR6sTz+2WrKK+x6D2NMcLDAOI4fB7+nVGPwu6i0nLvfW8ozM9cyvFsyk2/vT9PEmJ9tExMZzkPDOrJmZwGTM+30XWNMcLDAOI7/Xfnt3eD3zoNFXP3aAj5dvoP7L+jAi9f2ICYy3OO2F3VtRnrL+vzjy7UUFNmNm4wxgc8C4wSudw9+Z23xPPhdXqEszNnL/01dydAX57Jx92HGjUxnzKC2eL4vlIuI8Mjwzuw5XMIrszf6qnxjjKkxPputNlQM757MX6ZnM2nRVtLTGgCukMjcvI/pK/L4fOVO8guKiYkMY3DHxtx9bns6NK3j1Xt3a16Py09PYcK8TdzQtwWpDeJ8+VWMMeaUWGCcQHx0BJf0SObDrFyGd09mzprdzKgUEoM6NOaibs0Y3LExcVHV350PXNCRz1fs5MnPs3n1hl4++AbGGFMzLDC8cF2fFryTsZVb3lhMdISrJzGsqysk4qNPbRc2TYzhjrPb8PxX68jI2Uvf1g1rqGpjjKlZEkrTeKenp2tmZqZP3ntSxlbqxETUSEgc62hJOYP/MYeGCVFMGzOQsLCqxz6MMaYmiUiWqqZ7s60Nenvp+r4tXDPZ1nBYAMRGhfPghR1Zuf0QHy2xmzcZYwKTBUaAGNE9mR6p9Xh65lqOFJc5XY4xxvyCBUaA+PE02/yCYv79rZ1ma4wJPDboHUBOb1GfEd2TGTc3hzARmibG0LhONE3qun42TIgm3MY3jDEOscAIMA9e2JFVOw7y4tfrf/FamECjBFeAtGgQxx8v6kRKvVgHqjTG1EY+CwwRSQUmAk2BCmCcqr7ofu0u4E6gDJiuqg94aD8UeBEIB8ar6lO+qjWQJNeL5et7z6GkrII9h4vZXVDMrkNF7C4oZvehInYfKmZXQRGz1+5m16Ei3r+9v/U6jDF+4cseRhlwr6ouEZE6QJaIzAKaAJcA3VS1WEQaH9tQRMKBV4DzgVxgsYhMU9XVPqw3oERFhJFcL5bkKnoQH/+Qyz3vL+O1uRv57Tlt/VydMaY28tmgt6rmqeoS93IBkA2kAL8BnlLVYvdruz007wNsUNUcVS0B3sMVMsbt0h4pXNS1Gc/PWsfK7QedLscYUwv45SwpEUkDegIZQHvgTBHJEJFvRaS3hyYpQOV5v3Pd64ybiPDXy7pQPy6Ke95fSlFpudMlGWNCnM8DQ0QSgI+Asap6CNdhsPpAP+B+YLL8clpXTwflPV6SLiKjRSRTRDLz8/NrsPLAVy8uimev6s763Yd5+ou1TpdjjAlxPg0MEYnEFRbvqOoU9+pcYIq6LMI1IN7omKa5QGql582BHZ4+Q1XHqWq6qqYnJSXV7BcIAme1T+LmM9KY8P0m5q23W74aY3zHZ4Hh7jW8DmSr6nOVXpoKDHZv0x6IAo79TbcYaCcirUQkCrgWmOarWoPdH4Z2pE1SPPd9sKxat5NV1WrfftYYU3v5socxABgJDBaRpe7HMGAC0FpEVuIazL5JVVVEkkVkBoCqluE67XYmrsHyyaq6yoe1BrXYqHBeuKYnew4X86dPVnrVZmP+YX41YRHdH/+S68YtZMHGvV7dVdAYU3vZbLUh5OVv1vPsl+t48doeXNLD8zkCR4rL+Oc365kwbxMxEeFcfnrKT/f36J1Wn9+d246BbRsd926BxpjQUZ3Zai0wQkhZeQVXv7aA9bsPM3PsWT+7hkNV+XR5Hn+dvppdh4q5qldzHhjakaQ60RSVlvP+4m38+9uN5B0sokdqPX53blsGdWhswWFMiLPAqMW27D3ChS9+R4/Uevz31r6EhQlrdh7i0U9WkbFpH11S6vL4JV04vUX9X7QtLivnw6xcXp29ke0HjtIlpS53DW7H+Z2a2D06jAlRFhi13HuLtvLglBX8/vz27C8sYeKCLdSJieD+Czpwbe8WJ5xKpLS8go+XbOeVORvYsreQrimJ/Pe2viTGRvrpGxhj/KU6gWGTD4aga3qn8lX2bp6btQ4RuL5PC+4b0oH68VFetY8MD+Pq3qlcfnoKH2bl8uCUFXyQuY3bzmzt48qNMYHMAiMEiQh/v6IrL8+O5YrTm9MlJfGk3iciPIxr+7Tgw6xc3l64hVEDWtmhKWNqMbuBUohqmBDNo8NPO+mwqGxk/5Zs2VvIt+tr15X0xpifs8AwJ3Rhl2Y0Sohm4vzNTpdijHGQBYY5oaiIMK7vk8qcdfls2XvE6XKMMQ6xwDBeub5vS8JE+O/CLU6XYoxxiAWG8UrTxBguOK0JkzNzOVpiU6kbUxtZYBiv/ap/GgePljJt2XanSzHGOMACw3itb6sGdGhSh7fmb7GJCo2phSwwjNdEhJH9W7I67xBLtu53uhxjjJ9ZYJhquaxnCnWiI5i4wAa/jaltLDBMtcRHR3BFr+bMWJHH7oIip8sxxviRBYaptpH9W1Jarry3aJvTpRhj/MgCw1Rbm6QEzmzXiEkZWyktr3C6HGOMn1hgmJPyq/5p7DxUxKzVu5wuxRjjJxYY5qQM7tiYlHqxTFyw2elSjDF+YoFhTkp4mHBjv5YszNnH2p0FTpdjjPEDCwxz0q7pnUpURBhvL9zsdCnGGD+wwDAnrUF8FMO7JTNlyXYOFZU6XY4xxscsMMwpuemMlhSWlDMlK9fpUowxPuazwBCRVBGZLSLZIrJKRO52r39MRLaLyFL3Y1gV7TeLyAr3Npm+qtOcmm7N69E9tR4TF2yhqNRmsTUmlPmyh1EG3KuqnYB+wBgR6ex+7XlV7eF+zDjOewxyb5PuwzrNKbrjrNbk7DnCsH9+R9aWfU6XY4zxEZ8FhqrmqeoS93IBkA2k+OrzjHMu7NqMiaP6UFxawZX/XsATn622e2YYE4L8MoYhImlATyDDvepOEVkuIhNEpH4VzRT4UkSyRGT0cd57tIhkikhmfn5+jdZtvHdW+yRm3nMWN/RtwevzNjH0xbkszNnrdFnGmBokvr6vgYgkAN8Cf1XVKSLSBNiDKxCeAJqp6igP7ZJVdYeINAZmAXep6tzjfVZ6erpmZtpwh9Pmb9zDgx+tYOu+Qn7VvyV/GNqR+OgIp8syxnggIlneHvb3aQ9DRCKBj4B3VHUKgKruUtVyVa0A/gP08dRWVXe4f+4GPq5qOxN4zmjTiC/GnsktA9J4e+EWLnhhLvPW73G6LGPMKfLZn30iIsDrQLaqPldpfTNVzXM/vQxY6aFtPBCmqgXu5SHA476q1dS8uKgIHh1+Ghd1bcYDHy7nxtczuKhbM1o1jCc+OoKE6HD3T9cjPjqChJgImtaNsd6IMQHKZ4ekRGQg8B2wAvhxStM/AtcBPXAdktoM3K6qeSKSDIxX1WEi0hpXrwJcoTZJVf96os+0Q1KBqai0nOdnreP9zG0cOlpKxXH+yTWIj+KzuwaSXC/WfwUaU4tV55CUz8cw/MkCI/CpKkWlFRwuLuNwcRlHKv3cX1jK/01dyRltGjL+pnRcnVRjjC9VJzCs72/8SkSIjQonNiqcpDrRv3j9QGEJf5mezfQVeVzcLdmBCo0xVbGpQUxAuWVAK7o1T+Sxaas5WGjzUxkTSCwwTEAJDxOeurwb+wtL+NuMbKfLMcZUYoFhAk7n5Lr8+szWvJ+5jfkb7XRcYwKFBYYJSGPPa0fLhnH8ccoKm9TQmABhgWECUkxkOE9e1pXNewv559frnS7HGIMFhglgZ7RtxFW9mjNubg6rdxxyuhxjaj0LDBPQHr6oE/XiInloynLKj3fFnzHG5ywwTECrFxfFI8NPY1nuQd6cv9npcoyp1SwwTMAb3q0Zgzok8Y8v15K7v9DpcoyptSwwTMATEf5yWVcA/jR1JaE0nY0xwcQCwwSFlHqx3DekA3PW5jNt2Q6nyzGmVrLAMEHjpjPS6J5aj8c/Xc3+IyVOl2NMrWOBYYKGa9qQrhw8WspfbdoQY/zOAsMElU7N6jL6rNZ8mJXL/A02bYgx/mSBYYLO785tR1rDOB762KYNMcafLDBM0ImJDOdvl3Vli00bYoxfWWCYoHRG20Zc6Z42JDvPpg0xxh8sMEzQenhYJxJjI3lwygqbNsQYP7DAMEGrfnwUjwzvzLJtB3h7wWanyzEm5FlgmKA2onsyZ7dP4pmZa9lx4KjT5RgT0iwwTFATEf5yaRcqFP7Ppg0xxqe8CgwRuVtE6orL6yKyRESGnKBNqojMFpFsEVklIne71z8mIttFZKn7MayK9kNFZK2IbBCRB6v/1UxtkdogjnuHtOfrNbuZsWKn0+UYE7K87WGMUtVDwBAgCbgFeOoEbcqAe1W1E9APGCMind2vPa+qPdyPGcc2FJFw4BXgQqAzcF2ltsb8ws1npNE1JZFHp63iYGGp0+UYE5K8DQxx/xwGvKGqyyqt80hV81R1iXu5AMgGUrz8vD7ABlXNUdUS4D3gEi/bmlooIjyMJy/vyv7CEp76wqYNMcYXvA2MLBH5EldgzBSROkCFtx8iImlATyDDvepOEVkuIhNEpL6HJinAtkrPc6kibERktIhkikhmfn6+tyWZENQlJZFbB7bi3UXbyMjZ63Q5xoQcbwPjVuBBoLeqFgKRuA5LnZCIJAAfAWPdh7X+BbQBegB5wD88NfOwzuNopqqOU9V0VU1PSkrypiQTwsae147UBrHc9e4PvLtoKyVlXv9dY4w5AW8Doz+wVlUPiMiNwJ+AgydqJCKRuMLiHVWdAqCqu1S1XFUrgP/gOvx0rFwgtdLz5oDdBMGcUFxUBP+6oRfNEmN4aMoKznlmNhMXbLY5p4ypAd4Gxr+AQhHpDjwAbAEmHq+BiAjwOpCtqs9VWt+s0maXASs9NF8MtBORViISBVwLTPOyVlPLdUlJZOqYAbw1qg/J9WJ55JNVnPX0bF6ft4mjJRYcxpwsbwOjTF0nuF8CvKiqLwJ1TtBmADASGHzMKbRPi8gKEVkODALuARCRZBGZAaCqZcCdwExcg+WTVXVVdb+cqb1EhLPbJ/HBHf2Z9Ou+tE6K54nPVnPm09/w7283cqS4zOkSjQk64s2FTiLyLfAFMAo4E8gHlqpqV9+WVz3p6emamZnpdBkmQC3evI9/fr2e79bvoV5cJGPPbcfNA1o5XZYxp0RVcR3QOTkikqWq6d5s620P4xqgGNf1GDtxnbH0zEnWZ4wjeqc14O1b+/Lxb8+ga0oij326mldmb3C6LGNOyatzNnLj+Ay/nODhVWC4Q+IdIFFELgaKVPW4YxjGBKqeLerz5i19uKRHMs/MXMub329yuiRjToqqMmVJLqXlFURF+H6mJ2+nBrkaWARcBVwNZIjIlb4szBhfCg8Tnr2qO0M6N+GxT1czefG2EzcyJsCs3H6IjflHuLSnt9dEn5oIL7d7GNc1GLsBRCQJ+Ar40FeFGeNrkeFhvHR9T349MYs/TFlOdGQYl/Twz/94xtSEqUu3ExUexrAuzU68cQ3wtg8T9mNYuO2tRltjAlZ0RDiv3diL3mkN+P3kZXy5yiYvNMGhvEL5dNkOzumQRGJcpF8+09tf+l+IyEwRuVlEbgamA7+YNNCYYBQbFc6Em3vTNSWROyf9wNx1NsWMCXwLNu5ld0Gx3w5HgfeD3vcD44BuQHdgnKr+wZeFGeNPCdERvHVLH9o0TmD025k2F5UJeFOXbqdOdASDOzb222d6fVhJVT9S1d+r6j2q+rEvizLGCYlxkbx9ax9S6sVy61uZLN12wOmSjPGoqLScL1buZGiXpsREhvvtc4876C0iBXie9E8AVdW6PqnKGIc0Sojmndv6cfVrC7hpwiKevao7sZHhHC0t52hpOUUl5T8tHy0pp6i0nL6tGzC4YxOnSze1yFfZuzhcXMZlfjwcBScIDFU90fQfxoScpokxvHNbX65+bQG/nnj8mQPCBN5dtJUFD51LfLS3Jx0ac2qm/rCDJnWj6du6oV8/1/6FG+NBaoM4pv/uTFZsP0hsZLjrERVGzE/L4cREhLMs9wCXvTqfDzK32TQjxi8OFJbw7brd3HxGGuFhJz8lyMmwwDCmCg3iozi7/fHvsdKzRX16tazPhO83M7K///8HNrXP9BV5lJarI9cM2bUUxpyi2wa2Yuu+Qmattms4jO998sMO2jZO4LRk/w8hW2AYc4qGnNaU1AaxjP/O5qQyvpW7v5BFm/dxaY/kU5qh9mRZYBhzisLDhFEDWpG5ZT8/bN3vdDkmhH2y1HXjUaemsLHAMKYGXJ2eSp2YCMbPs16G8Q1V5ZOl20lvWZ/UBnGO1GCBYUwNiI+O4Pq+Lfh8RR7b9hU6XY4JQdl5BazbdZhL/HztRWUWGMbUkJvPSCNMhDfnb3a6FBOCPlm6nYgw4aKu/pmZ1hMLDGNqSLPEWC7u1oz3F2/jUFGp0+WYEFJRoUxbtoOz2yfRID7KsTosMIypQbed2ZrDxWW8v8huyGRqTsamfeQdLHL0cBRYYBhTo7qkJNKvdQPe+H4TZeW+v8eyqR0+Wbqd+Khwzu/k7JxlFhjG1LDbBrZmx8EiZqy0C/nMqSsqLWf6ijwuOK0psVH+m5nWE58FhoikishsEckWkVUicvcxr98nIioijapoXy4iS92Pab6q05iaNrhjY1o3imf8dzmoeprs2RjvzVm7m4KiMscPR4FvexhlwL2q2gnoB4wRkc7gChPgfGDrcdofVdUe7scIH9ZpTI0KCxNGDWzF8tyDLN5sF/KZUzP1hx00SohmQBv/zkzric8CQ1XzVHWJe7kAyAZ+jMjngQfwfK8NY4LeFac3p35cJOO/y3G6FBPEDh4t5Zs1uxnevRkR4c6PIPilAhFJA3oCGSIyAtiuqstO0CxGRDJFZKGIXHqc9x7t3i4zP9/uxWwCQ2xUODf2a8ms7F1s2nPE6XJMkPp8RR4l5RVc6tBUIMfyeWCISALwETAW12Gqh4FHvGjaQlXTgeuBF0SkjaeNVHWcqqaranpS0vGnojbGn0b2b0lkWBhvfG/ThZjqKy2v4LW5OXRsWoduzROdLgfwcWCISCSusHhHVacAbYBWwDIR2Qw0B5aISNNj26rqDvfPHGAOrh6KMUGjcZ0YRvRI5oPMXA4UljhdjgkyH2blsmnPEe4d0sGRmWk98eVZUgK8DmSr6nMAqrpCVRurapqqpgG5wOmquvOYtvVFJNq93AgYAKz2Va3G+MptZ7biaGk572Qc7/wOY36uqLScF75ax+kt6nFep8ZOl/MTX/YwBgAjgcGVTo8dVtXGIpIuIuPdTzsBmSKyDJgNPKWqFhgm6HRsWpdBHZJ4dfYGm5TQeG3igs3sOlTMA0M7BkzvAkBC6Tzx9PR0zczMdLoMY34md38hQ1/4ji4pdZl0Wz/C7Dau5jgOFZVy1tOz6da8HhNH9fH554lIlnu8+IScP0/LmBDXvH4cj1zcmYU5+3hrwWanyzEBbvzcHA4UlvLABR2cLuUXLDCM8YOr0pszuGNjnvp8DRvzDztdjglQ+QXFjJ+3iYu6NaNLSmCcGVWZBYYxfiAiPHV5V2Iiw7l38jKbmNB49MrsDRSXVXDv+e2dLsUjCwxj/KRx3RieuLQLS7cd4LW5dgW4+blt+wqZlLGVq9Ob0zopwelyPLLAMMaPhndrxkVdm/HCV+vIzjvkdDkmgLzw1XoQ+N257ZwupUoWGMb4kYjwxKVdSIyN4veTl1FSZoemDKzfVcDHP+RyU/+WNEuMdbqcKllgGONnDeKjePLyrmTnHeKlb9Y7XY4JAM9+uZa4qAh+c05bp0s5LgsMYxxwfucmXNmrOa/O2cjSbQecLsc4aOm2A8xctYvRZ7V29H7d3rDAMMYhjwzvTJM60fx+8lKKSsudLsc45Okv1tAwPopRA1s5XcoJWWAY45C6MZE8fWV3cvKP8MzMtU6XYxwwb/0e5m/cy5hBbUmIjnC6nBOywDDGQQPbNWJkv5ZM+H4TC3P2Ol2O8SNV5emZa0ipF8sN/Vo4XY5XLDCMcdhDwzrSskEcY99byp7DxU6XY/xk5qqdLM89yNjz2hEdEe50OV6xwDDGYXFREbxyw+nsLyzhzklL7CrwWqCiQnnhq/W0Torn8tObO12O1ywwjAkApyUn8rfLurIwZ5+NZ9QCX2XvYs3OAu4c1JbwIJq92ALDmABxRa/mjOzXktfm5jBjRZ7T5RgfUVVe+mYDLRvGMaJ7stPlVIsFhjEB5P8u7kzPFvW4/4NlbNhd4HQ5xgfmrMtnxfaD/PacNkSEB9ev4OCq1pgQFxURxqs3nE5sVDi3v53F4eIyp0syNUhVeenr9aTUi+WynsEzdvEjCwxjAkyzxFheuu50Nu8t5P4PlhFKd8Ws7eZv3MuSrQe445w2REUE36/f4KvYmFqgf5uGPDi0I5+v3Mk4mwo9ZPzz6/U0qRvNVb2Cr3cBFhjGBKzbzmzFRV2b8fcv1jB/wx6nyzGnaNGmfWRs2sftZ7UhJjI4rrs4lgWGMQFKRPj7ld1onZTAXe/+wI4DR50uyZyCl75ZT6OEKK7rExxXdXtigWFMAEuIjuDfN/aiuKyC37yzhOIym6QwGP2wdT/frd/Dr89sTWxUcPYuwIeBISKpIjJbRLJFZJWI3H3M6/eJiIpIoyra3yQi692Pm3xVpzGBrm3jBJ69qhvLth3gtrcy2bL3iNMlmWp66ZsN1I+L5MZ+LZ0u5ZT4sodRBtyrqp2AfsAYEekMrjABzge2emooIg2AR4G+QB/gURGp78NajQloQ7s044lLu7Bky37Of24uz85cS2GJnXIbDFZuP8g3a3Zz25mtiQ+CGWmPx2eBoap5qrrEvVwAZAMp7pefBx4Aqjpf8AJglqruU9X9wCxgqK9qNSYYjOzXkm/uO4dhXZvy8uwNnPePb5m+PM9Ouw1wL32znroxEfyqf3D3LsBPYxgikgb0BDJEZASwXVWXHadJCrCt0vNc/hc2xtRaTerG8MK1Pfngjv4kxkUxZtISbhifwbpddlV4IMrOO8TMVbu4ZUAr6sREOl3OKfN5YIhIAvARMBbXYaqHgUdO1MzDOo9/RonIaBHJFJHM/Pz8U6rVmGDRO60Bn901kCcuOY1VOw5x4Yvf8finqzlUVOp0aaaSl2dvICE6glEDAv9uet7waWCISCSusHhHVacAbYBWwDIR2Qw0B5aISNNjmuYCqZWeNwd2ePoMVR2nqumqmp6UlFTTX8GYgBUeJozsn8bs+87hmt6pvDF/E4OfnWPXbASIDbsLmLEij1/1b0liXPD3LsC3Z0kJ8DqQrarPAajqClVtrKppqpqGKxhOV9WdxzSfCQwRkfruwe4h7nXGmGM0iI/ib5d1ZdqYgdSNieSBj5ZTUmb31HDaK7M3EhMRzq1BcK9ub/myhzEAGAkMFpGl7sewqjYWkXQRGQ+gqvuAJ4DF7sfj7nXGmCp0bZ7IYyNOI3f/Ud5d5PEEROMnm/cc4ZOl27mxXwsaJkQ7XU6N8dk5Xqo6D89jEZW3Sau0nAncVun5BGCCr+ozJhSd2a4R/Vs35KVv1nNlr+ZBfxpnsHpm5lqiIsL49VmtnS7oi1FuAAAQkklEQVSlRtmV3saEEBHhgaEd2HO4hNfnbXK6nFrp+w17mL4ij9+e05bGdWKcLqdGWWAYE2J6tqjP0NOaMm5uDnsPFztdTq1SWl7BY9NWkdogltEh1rsACwxjQtJ9F7SnsKSMV+dsdLqUWmXigi2s332YRy4+LWhnpD0eCwxjQlDbxnW4qlcqby/YQu7+QqfLqRXyC4p5YdY6zm6fxHmdGjtdjk9YYBgTou4+rx0IvPDVeqdLqRX+/sUaisrKeXR4Z1xXFYQeCwxjQlRyvVhu6t+SKUtybeoQH1uydT8fZuUyamArWiclOF2Oz1hgGBPCfntOW+KjInhm5lqnSwlZ5RXKo5+sokndaO4a3M7pcnzKAsOYEFY/Porbz27NrNW7yNqy3+lyQtLkzG2s2H6QPw7rREKIX/digWFMiBs1sBWNEqL5+xdrbCr0GnagsISnv1hDn7QGjOie7HQ5PmeBYUyIi4uK4O5z27Jo0z7mrLMZnWvSc7PWcfBoKY+NOC1kB7ors8Awpha4pncLWjSI4+kv1lJRYb2MmrB6xyH+u3ALN/RtSefkuk6X4xcWGMbUAlERYdw7pD3ZeYf4dLnHOwWYalBVHpu2isTYSO4d0t7pcvzGAsOYWmJ4t2Q6NavLP75cZ9Ofn6Jpy3awaPM+7r+gI/Xiopwux29Ce0jfGPOTsDDXxIS3vLGYG8dn0L9NQ3qk1qNb80Svp+A+XFzGmrxDrM47xKY9R7i4WzN6tWzg48oDy+HiMv42I5suKXW5pnfqiRuEEAsMY2qRc9onMfa8dnyxcicvfbOeH4czUhvE0iO1Pt2bJ9IjtR6nJSdy4GgJq3ccYvWOQ2TvdP3cvPd/04xEhAlvzd/MmEFt+d257YgMD/0DFmt3FjD2/aXsLijm1Rt6ER4W+gPdlUkonWaXnp6umZmZTpdhTFA4UlzGiu0HWbbtAMtyD7B06wF2HCzyuG3LhnF0blbX9Uh2PRKiI/jzp6v5MCuX7qn1ePGaHqQ1ivfzt/CPigrl9XmbeGbmWurGRvDU5d04r3MTp8uqESKSparpXm1rgWGM+dHuQ0Usyz3Iyu0HaZgQRedmdenQtA51Yqq+J/X05Xk8NGU5ZRXKo8M7c3V6akidYpq7v5B7Jy8jY9M+zu/chCcv70qjELqLngWGMcavdhw4yu8nL2Vhzj6GntaUJy/vSv344B4MVlWmLNnOY9NWUaHKoyNO46pezUMqDKF6gRH6Bx2NMT6XXC+WSbf146ELO/L1ml0MfXEu89bvcbqsk7bvSAm/+e8S7v1gGZ2a1eWLsWeFXM/pZFhgGGNqRFiYcPvZbfj4twOIj47gxtcz+MtnqykuK3e6tGqZvWY3Q56fy9drdvHghR15d3Q/UhvEOV1WQLDAMMbUqC4piUy/60xu7NeC8fM2ccfbWRSVBkdoTJi3iVveXEyjhCg+GTOQO85uU+vOhDoeCwxjTI2LjQrnL5d25W+XdWX22nzu+G/gh8YnS7fz+GerueC0JkwdM6DWTPdRHT4LDBFJFZHZIpItIqtE5G73+idEZLmILBWRL0XE4xSPIlLu3mapiEzzVZ3GGN+5vm8Lnrq8K3PW5jM6gHsa363P574PltG3VQNevLZnSN6Puyb4sodRBtyrqp2AfsAYEekMPKOq3VS1B/AZ8EgV7Y+qag/3Y4QP6zTG+NC1fVrw9BXd+G59Pr+emBlwobE89wB3vJ1Fm6QExv0q3cLiOHwWGKqap6pL3MsFQDaQoqqHKm0WD4TOeb3GGI+u7p3K01d0Y96GPdz61mKOlgRGaGzac4Rb3lhMvbgo3hrVh8TYqq83MX4awxCRNKAnkOF+/lcR2QbcQNU9jBgRyRSRhSJyqT/qNMb4zlXpqTx7ZXfmb9zLqDcXU1hS5mg9uwuK+NWEDBR4+9Y+NKkb42g9wcDngSEiCcBHwNgfexeq+rCqpgLvAHdW0bSF+2KS64EXRKRNFe8/2h0smfn5dnMYYwLZFb2a89zV3cnY5GxoFBSVcvOExew9XMKEm3vTOinBkTqCjU8DQ0QicYXFO6o6xcMmk4ArPLVV1R3unznAHFw9FE/bjVPVdFVNT0pKqpG6jTG+c1nP5jx/TQ8WbdrHzW8s5kixf0OjuKyc0ROzWLergH/d2IseqfX8+vnBzJdnSQnwOpCtqs9VWt+u0mYjgDUe2tYXkWj3ciNgALDaV7UaY/zrkh4pvHhtT7K27OfmNxYxe+1utuw9Qlm5b+/TUV6h3PP+Uhbk7OWZq7pxdnv7I7M6fDm9+QBgJLBCRJa61/0RuFVEOgAVwBbgDgARSQfuUNXbgE7AayJSgSvUnlJVCwxjQsjw7smEiTD2/R+45Y3FAESGCy0axNGqUQKtk+Jp1cj1aJOUQFKdU5vwT1X586ermLFiJw8P68RlPZvXxNeoVWzyQWOMow4WlrIhv4CN+UfYtOcIm378uffIz+4MeF6nxtx9bnu6Nk+s1vurKt9v2MvzX60ja8t+Rp/Vmj8O61TTXyNo2Wy1xpigV16h5B08Sk7+ETI37+OtBVs4eLSUwR0bc/e57ejuxdjD/I17eH7WOhZv3k+zxBjGDGrL9X1aEGbTffzEAsMYE3IKikp5a/5mxs/bxIHCUgZ1SOLu89p7HLRemLOX52etI2PTPprUjWbMoLZc0zuV6Ai7KO9YFhjGmJBVUFTKxAVb+M93ORwoLOXs9kncfV47Tm9Rn0Wb9vH8rHUsyNlLUp1oxpzThmv7tLCrt4/DAsMYE/IOF5cxccFm/jM3h/2FpbRqFM+mPUdolBDNb85pww19LSi8YYFhjKk1jhSX8fbCLXy5aifDujbjhr4tiY2yoPCWBYYxxhiv2C1ajTHG1DgLDGOMMV6xwDDGGOMVCwxjjDFescAwxhjjFQsMY4wxXrHAMMYY4xULDGOMMV4JqQv3RCQf1z02KksEDp7EukbAnhot8MQ81eXL9t5sf7xtqvua7Xfvt7f9XvPtbb97rqGlqnp3JylVDekHMO5k1gGZgVCrL9t7s/3xtqnua7bfbb/bfg/u/V4bDkl9egrr/O1Ua6hue2+2P9421X3N9rv329t+r/n2tt9dTrqGkDokVZNEJFO9nF/F1Bzb786w/e6MYNvvtaGHcbLGOV1ALWX73Rm2350RVPvdehjGGGO8Yj0MY4wxXrHAMMYY4xULDGOMMV6xwDhJIhIvIlkicrHTtdQWItJJRP4tIh+KyG+crqe2EJFLReQ/IvKJiAxxup7aQkRai8jrIvKh07X8qNYFhohMEJHdIrLymPVDRWStiGwQkQe9eKs/AJN9U2XoqYn9rqrZqnoHcDUQNKciOqmG9vtUVf01cDNwjQ/LDRk1tN9zVPVW31ZaPbXuLCkROQs4DExU1S7udeHAOuB8IBdYDFwHhANPHvMWo4BuuC7pjwH2qOpn/qk+eNXEflfV3SIyAngQeFlVJ/mr/mBVU/vd3e4fwDuqusRP5QetGt7vH6rqlf6q/XginC7A31R1roikHbO6D7BBVXMAROQ94BJVfRL4xSEnERkExAOdgaMiMkNVK3xaeJCrif3ufp9pwDQRmQ5YYJxADf17F+Ap4HMLC+/U1L/3QFPrAqMKKcC2Ss9zgb5VbayqDwOIyM24ehgWFienWvtdRM4BLgeigRk+rSy0VWu/A3cB5wGJItJWVf/ty+JCWHX/vTcE/gr0FJGH3MHiKAsMF/Gw7oTH6lT1zZovpVap1n5X1TnAHF8VU4tUd7//E/in78qpNaq73/cCd/iunOqrdYPeVcgFUis9bw7scKiW2sT2uzNsvzsj6Pe7BYbLYqCdiLQSkSjgWmCawzXVBrbfnWH73RlBv99rXWCIyLvAAqCDiOSKyK2qWgbcCcwEsoHJqrrKyTpDje13Z9h+d0ao7vdad1qtMcaYk1PrehjGGGNOjgWGMcYYr1hgGGOM8YoFhjHGGK9YYBhjjPGKBYYxxhivWGAYx4jIYT98xggvp6uvyc88R0TOOIl2PUVkvHv5ZhF5uearqz4RSTt2mm4P2ySJyBf+qsk4wwLDBD33tNEeqeo0VX3KB595vHnYzgGqHRjAH4GXTqogh6lqPpAnIgOcrsX4jgWGCQgicr+ILBaR5SLy50rrp7rvbLhKREZXWn9YRB4XkQygv4hsFpE/i8gSEVkhIh3d2/30l7qIvCki/xSR+SKSIyJXuteHicir7s/4TERm/PjaMTXOEZG/ici3wN0iMlxEMkTkBxH5SkSauKe0vgO4R0SWisiZ7r++P3J/v8WefqmKSB2gm6ou8/BaSxH52r1vvhaRFu71bURkofs9H/fUYxPXnSGni8gyEVkpIte41/d274dlIrJIROq4exLfuffhEk+9JBEJF5FnKv23ur3Sy1OBGzz+BzahQVXtYQ9HHsBh988hwDhcs3mGAZ8BZ7lfa+D+GQusBBq6nytwdaX32gzc5V7+LTDevXwzrpstAbwJfOD+jM647k0AcCWu6dLDgKbAfuBKD/XOAV6t9Lw+/5st4TbgH+7lx4D7Km03CRjoXm4BZHt470HAR5WeV677U+Am9/IoYKp7+TPgOvfyHT/uz2Pe9wrgP5WeJwJRQA7Q272uLq6Zq+OAGPe6dkCmezkNWOleHg38yb0cDWQCrdzPU4AVTv+7sofvHja9uQkEQ9yPH9zPE3D9wpoL/E5ELnOvT3Wv3wuUAx8d8z5T3D+zcN03w5Op6rp/yWoRaeJeNxD4wL1+p4jMPk6t71dabg68LyLNcP0S3lRFm/OAziI/zW5dV0TqqGpBpW2aAflVtO9f6fu8DTxdaf2l7uVJwLMe2q4AnhWRvwOfqep3ItIVyFPVxQCqeghcvRHgZRHpgWv/tvfwfkOAbpV6YIm4/ptsAnYDyVV8BxMCLDBMIBDgSVV97WcrXTdMOg/or6qFIjIH121xAYpUtfyY9yl2/yyn6n/bxZWW5Zif3jhSafkl4DlVneau9bEq2oTh+g5Hj/O+R/nfdzsRryeAU9V1ItILGAY8KSJf4jp05Ok97gF2Ad3dNRd52EZw9eRmengtBtf3MCHKxjBMIJgJjBKRBAARSRGRxrj+et3vDouOQD8fff484Ar3WEYTXIPW3kgEtruXb6q0vgCoU+n5l7hmKQXA/Rf8sbKBtlV8znxcU2GDa4xgnnt5Ia5DTlR6/WdEJBkoVNX/4uqBnA6sAZJFpLd7mzruQfxEXD2PCmAkrntNH2sm8BsRiXS3be/umYCrR3Lcs6lMcLPAMI5T1S9xHVJZICIrgA9x/cL9AogQkeXAE7h+QfrCR7hubrMSeA3IAA560e4x4AMR+Q7YU2n9p8BlPw56A78D0t2DxKvxcBc1VV2D6xaodY59zd3+Fvd+GAnc7V4/Fvi9iCzCdUjLU81dgUUishR4GPiLqpYA1wAvicgyYBau3sGrwE0ishDXL/8jHt5vPLAaWOI+1fY1/tebGwRM99DGhAib3twYQEQSVPWwuO6jvAgYoKo7/VzDPUCBqo73cvs44Kiqqohci2sA/BKfFnn8euYCl6jqfqdqML5lYxjGuHwmIvVwDV4/4e+wcPsXcFU1tu+Fa5BagAO4zqByhIgk4RrPsbAIYdbDMMYY4xUbwzDGGOMVCwxjjDFescAwxhjjFQsMY4wxXrHAMMYY4xULDGOMMV75f5zD+h63k3LQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The beginning loss is even lower than when we initialized at -3. It also looks like we need to pick a lower learning rate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "82223eef07e04f8a929b27ed689b9216",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      19.360822  17.005886 \n",
      "    1      17.448353  15.66676                                                                                         \n",
      "    2      16.08572   15.372393                                                                                        \n",
      "    3      14.983382  14.701052                                                                                        \n",
      "    4      14.041856  14.399405                                                                                        \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[14.399405]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lr/2, 1, cycle_len=5, use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And indeed our final result is slightly better."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('bias-6')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "-6.0423\n",
       "[torch.cuda.FloatTensor of size 1 (GPU 0)]"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outConv1 = learn.model[8].out.oconv1\n",
    "outConv1.bias.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Again, the values of our bias have barely changed."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So an initial negative bias seem to help a lot. How do we pick a good value? Looking at all the plots from our learning rate finder, I had the idea to try to find a value that minimizes the initial loss. The function below computes the loss for the first training mini-batch with bias ranging from -10 to 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_loss(min_b=-10,max_b=0,n=100):\n",
    "    test_biases = np.linspace(min_b,max_b,n)\n",
    "    losses = np.zeros(n)\n",
    "    x,y=next(iter(md.trn_dl))\n",
    "    x,y = V(x),V(y)\n",
    "    nb = learn.model[8].out.oconv1.bias.size(0)\n",
    "    fp = FloatProgress(min=0,max=n)\n",
    "    display(fp)\n",
    "    for i in range(n):\n",
    "        learn.model[8].out.oconv1.bias = torch.nn.Parameter(torch.Tensor([test_biases[i]]*nb).cuda())\n",
    "        z = learn.model(x)\n",
    "        losses[i] = ssd_loss(z,y)\n",
    "        fp.value += 1\n",
    "    return test_biases,losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_reg4 = SSD_Head(k,0)\n",
    "models = ConvnetBuilder(f_model, 0, 0, 0, custom_head=head_reg4)\n",
    "learn = ConvLearner(md, models)\n",
    "learn.opt_fn = optim.Adam\n",
    "learn.crit = ssd_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8987171808424618bce0ab886d8301f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>FloatProgress</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "FloatProgress(value=0.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "b,l = test_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmcXXV9//HX5y6zJbNkyCSELEyAsIQthCREUaGAAtoaFVHsIqUKtT9al9bW7fer2l99VGkr1traHy0ItipSAUkVKItgAYWQnSyEDFknk2QmmS2T2e7y+f1xz8QxTDKTzNx77vJ+Ph73ce/93nPu/Zws73PO93zPOebuiIhI8YqEXYCIiGSXgl5EpMgp6EVEipyCXkSkyCnoRUSKnIJeRKTIKehFRIqcgl5EpMgp6EVEilws7AIApk6d6o2NjWGXISJSUFatWnXA3RtGmy4vgr6xsZGVK1eGXYaISEExs51jmU5dNyIiRU5BLyJS5BT0IiJFTkEvIlLkFPQiIkVOQS8iUuQU9CIiRU5BLyISkm889RovNB3I+u8o6EVEQnCoP8E/PL2VlTs6sv5bCnoRkRCsb+7CHRbMqcv6bynoRURCsGZXZkt+wSwFvYhIUVqzq5MzGyZRWxXP+m8p6EVEcszdWbO7k0vmTMnJ740a9GZWYWYrzGydmW00sy8H7fea2XYzWxs8FgTtZmbfNLMmM1tvZguzvRAiIoVkd3sf7YcHuSQH/fMwtssUDwBXuXuPmcWB583sseCzP3f3Hx01/fXAvOBxGfDt4FlERIA1u4P++dm5CfpRt+g9oyd4Gw8efpxZlgHfDeZ7EagzsxnjL1VEpDis2dVJZTzKOdOrc/J7Y+qjN7Ooma0FWoEn3f2l4KOvBN0zd5pZedA2E9g9bPbmoE1ERMiMuLloVi2xaG4Ok47pV9w95e4LgFnAEjO7APgccC6wGKgHPhNMbiN9xdENZnabma00s5VtbW0nVbyISKHpT6TYtLc7Zwdi4QRH3bh7J/AscJ277w26ZwaA7wBLgsmagdnDZpsFtIzwXXe5+yJ3X9TQMOotD0VEisLGlm4SKc/ZgVgY26ibBjOrC15XAtcArw71u5uZAe8BNgSzLAc+HIy+WQp0ufverFQvIlJghk6UuiRHB2JhbKNuZgD3mVmUzIrhAXf/iZn9zMwayHTVrAU+Fkz/KPBOoAnoBW6Z+LJFRArTmt2dzKyrZFpNRc5+c9Sgd/f1wCUjtF91jOkduH38pYmIFJ+1uzpzcn2b4XRmrIhIjrR297Onsy+n3TagoBcRyZnVQf/8wtNzN+IGFPQiIjmzamcHZbEI559Wk9PfVdCLiOTIqp0dXDSzlvJYNKe/q6AXEcmBgWSKDXu6uTTH3TagoBcRyYkNe7oZTKVzekbsEAW9iEgOrN45dCA2tyNuQEEvIpITq3Z2MKe+imnVuTtRaoiCXkQky9ydVbs6QumfBwW9iEjWNXf00XZogIU5PiN2iIJeRCTLwjpRaoiCXkQky1bt7GBSWe7uKHU0Bb2ISJat3tXBgjl1Obuj1NEU9CIiWdQ7mGTz3kMsDGH8/BAFvYhIFq3Z1Ukq7aGNuAEFvYhIVq3Y3k7EUNCLiBSrl3e0c96MGqor4qHVoKAXEcmSRCrNml2dLG6sD7UOBb2ISJZs2NNFXyKloBcRKVYv72gHYPHc8PrnYQxBb2YVZrbCzNaZ2UYz+3LQPtfMXjKzrWb2QzMrC9rLg/dNweeN2V0EEZH8tGJ7B42nhHMhs+HGskU/AFzl7hcDC4DrzGwp8DXgTnefB3QAHwmm/wjQ4e5nAXcG04mIlJR02lm5sz30bhsYQ9B7Rk/wNh48HLgK+FHQfh/wnuD1suA9wedXm5lNWMUiIgWgqa2Hzt4Ei+cWQNADmFnUzNYCrcCTwOtAp7sng0magZnB65nAboDg8y7glIksWkQk363YnumfX1IIW/QA7p5y9wXALGAJcN5IkwXPI229+9ENZnabma00s5VtbW1jrVdEpCC8vKOdhupyTj+lKuxSTmzUjbt3As8CS4E6M4sFH80CWoLXzcBsgODzWqB9hO+6y90XufuihoaGk6teRCRPvby9nSWN9eRDz/VYRt00mFld8LoSuAbYDDwDvD+Y7GbgkeD18uA9wec/c/c3bNGLiBSr5o5eWrr6WdwY7rDKIbHRJ2EGcJ+ZRcmsGB5w95+Y2SbgfjP7a2ANcHcw/d3Av5tZE5kt+ZuyULeISN56aVvQPz83Pw5Pjhr07r4euGSE9m1k+uuPbu8HbpyQ6kRECtBL2w9SWxnn3FPDudHI0XRmrIjIBHtxWztL5tYTiYTfPw8KehGRCdXS2ceu9l6WnpEf3TagoBcRmVAvbT8IwGV5cKLUEAW9iMgEemlbOzUVMc6bURN2KUco6EVEJtCL2w6yZG490TzpnwcFvYjIhNnX1c+Og/nVPw8KehGRCfOr/nkFvYhIUXpxWzvV5THmn5Y//fOgoBcRmTAvbTvI4jzrnwcFvYjIhGjt7mfbgcMsPSN/hlUOUdCLiEyAX27L9M/n24FYUNCLiEyI57YeoK4qzvmn1YZdyhso6EVExsndeW5rG5efOTXv+udBQS8iMm5NrT3s7x7gLfOmhl3KiBT0IiLj9NzWAwC85SwFvYhIUXq+6QBzp05idn3494cdiYJeRGQcBpNpXtx2MG+35kFBLyIyLqt3ddA7mOKtedo/Dwp6EZFxeX7rAaIRY+mZ+Td+foiCXkRkHJ7b2saC2XXUVMTDLuWYRg16M5ttZs+Y2WYz22hmnwjav2Rme8xsbfB457B5PmdmTWa2xcyuzeYCiIiEpbN3kPV7uvK6fx4gNoZpksCfuftqM6sGVpnZk8Fnd7r73w2f2MzmAzcB5wOnAU+Z2dnunprIwkVEwvZC00Hcyev+eRjDFr2773X31cHrQ8BmYOZxZlkG3O/uA+6+HWgClkxEsSIi+eTZLa3UVMRYMLsu7FKO64T66M2sEbgEeClo+mMzW29m95jZlKBtJrB72GzNHH/FICJScNydn7/WxlvPbiAWze/DnWOuzswmAw8Cn3T3buDbwJnAAmAv8PdDk44wu4/wfbeZ2UozW9nW1nbChYuIhGnT3m5aDw1w5dkNYZcyqjEFvZnFyYT899z9IQB33+/uKXdPA//Kr7pnmoHZw2afBbQc/Z3ufpe7L3L3RQ0N+f8HJSIy3LNbMhuoV5yT//k1llE3BtwNbHb3rw9rnzFssvcCG4LXy4GbzKzczOYC84AVE1eyiEj4fr6ljfNPq2FadUXYpYxqLKNuLgd+D3jFzNYGbZ8HPmRmC8h0y+wA/hDA3Tea2QPAJjIjdm7XiBsRKSZdfQlW7ergj644M+xSxmTUoHf35xm53/3R48zzFeAr46hLRCRvPb/1AKm0c2UBdNuAzowVETlhhTKscoiCXkTkBBTSsMohhVGliEieKKRhlUMU9CIiJ+CZV1uBwhhWOURBLyJyAp7c3MqC2XUFMaxyiIJeRGSM9nf3s253J2+fPz3sUk6Igl5EZIye2rwfQEEvIlKsntq0nzn1VcybNjnsUk6Igl5EZAwODyR54fWDvH3+dDJXhikcCnoRkTF4bmsbg8k015xXWN02oKAXERmTJzbtp7YyzuLGKaNPnGcU9CIio0im0jzzaitXnTutYM6GHa7wKhYRybFVOzvo6E0U3GibIQp6EZFRPLlpP2XRCG8roMseDKegFxE5DnfnsQ37eMu8qUwuH8stPPKPgl5E5Dg27OlmT2cf111watilnDQFvYjIcTy2YS/RiPH2AhxWOURBLyJyDO7O4xv2sfSMeqZMKgu7nJOmoBcROYatrT1sO3CY6y6YEXYp46KgFxE5hsde2YcZXFugwyqHjBr0ZjbbzJ4xs81mttHMPhG015vZk2a2NXieErSbmX3TzJrMbL2ZLcz2QoiIZMNjG/Zy6ZwpTKspnGvPj2QsW/RJ4M/c/TxgKXC7mc0HPgs87e7zgKeD9wDXA/OCx23Atye8ahGRLNtx4DCv7jtU0KNthowa9O6+191XB68PAZuBmcAy4L5gsvuA9wSvlwHf9YwXgTozK+wOLhEpOY9v3AdQGkE/nJk1ApcALwHT3X0vZFYGwLRgspnA7mGzNQdtR3/XbWa20sxWtrW1nXjlIiJZ9F/rWrh4Vi2zplSFXcq4jTnozWwy8CDwSXfvPt6kI7T5Gxrc73L3Re6+qKGhME8rFpHi1NTaw8aWbt694A3bqAVpTEFvZnEyIf89d38oaN4/1CUTPLcG7c3A7GGzzwJaJqZcEZHsW76uBTP4rYuKo9d5LKNuDLgb2OzuXx/20XLg5uD1zcAjw9o/HIy+WQp0DXXxiIjkO3dn+do9vPnMUwp+tM2QsVyh53Lg94BXzGxt0PZ54KvAA2b2EWAXcGPw2aPAO4EmoBe4ZUIrFhHJovXNXew42Mv/uvKssEuZMKMGvbs/z8j97gBXjzC9A7ePsy4RkVAsX9dCWTTCtUUw2maIzowVEQmk0s5/rWvhynMaqK2Mh13OhFHQi4gEXtp2kNZDAywrktE2QxT0IiKBR9a2MKksytXnTRt94gKioBcRAfoGU/z0lb1cf+EMKuLRsMuZUAp6ERHgiU376BlIcsPCWWGXMuEU9CIiwI9WNTOzrpLL5taHXcqEU9CLSMnb19XPC00HuGHhTCKRY40mL1wKehEpeQ+v2UPa4X1F2G0DCnoRKXHuzoOrm1l0+hQap04Ku5ysUNCLSElb39xFU2sPN1xanFvzoKAXkRL34OpmymMR3lUkV6ociYJeREpW72CSh9fs4boLTqWmongueXA0Bb2IlKyfrNvLof4kv3PZ6WGXklUKehEpWd97aSfzpk1mceOUsEvJKgW9iJSkV5q7WNfcxe9cNofM/ZWKl4JeRErS91fspCIe4b1FOnZ+OAW9iJScQ/0JHlnbwrsvPq2orjt/LAp6ESk5P17bQu9gqugPwg5R0ItISXF3/uOXOzn/tBoumlUbdjk5oaAXkZLyy20H2bL/EDe/ubHoD8IOGTXozeweM2s1sw3D2r5kZnvMbG3weOewzz5nZk1mtsXMrs1W4SIiJ+PeF3ZQP6mMd198Wtil5MxYtujvBa4bof1Od18QPB4FMLP5wE3A+cE8/2xmxXWrFhEpWLvbe3lq834+tGR20d1F6nhGDXp3/x+gfYzftwy4390H3H070AQsGUd9IiIT5t9f3ImZ8btLS+Mg7JDx9NH/sZmtD7p2hk4rmwnsHjZNc9D2BmZ2m5mtNLOVbW1t4yhDRGR0vYNJ7l+xi+svOJUZtZVhl5NTJxv03wbOBBYAe4G/D9pHOrLhI32Bu9/l7ovcfVFDQ8NJliEiMjYPrd5Dd3+SWy5vDLuUnDupoHf3/e6ecvc08K/8qnumGZg9bNJZQMv4ShQRGZ902vnOC9u5cGYtC+cU93VtRnJSQW9mwy/c/F5gaETOcuAmMys3s7nAPGDF+EoUERmf/964j9fbDnPr284omSGVw8VGm8DMfgBcCUw1s2bgi8CVZraATLfMDuAPAdx9o5k9AGwCksDt7p7KTukiIqNzd771TBNzp07iXRcW781FjmfUoHf3D43QfPdxpv8K8JXxFCUiMlGe3dLGxpZu/vb9FxGNlN7WPOjMWBEpYu7ON3+2lZl1lbznkhEHAJYEBb2IFK1fvn6QNbs6+diVZxKPlm7cle6Si0jR+9YzTUyrLufGS4v/mvPHo6AXkaL08o52fvH6QW572xkldbmDkSjoRaQoffPprUydXFYy15w/HgW9iBSd1bs6eG7rAW596xlUlpX21jwo6EWkCH3z6a3UTyoruYuXHYuCXkSKytrdnTy7pY2PvnUuk8pHPVWoJCjoRaSo/OPTW6mrivPhNzWGXUreUNCLSNFYtbODp19t5aNvmctkbc0foaAXkaLg7vzNo5tpqC7nlsvnhl1OXlHQi0hReGLTflbu7OCT18xT3/xRFPQiUvCSqTRfe/xVzmiYxAcXzR59hhKjoBeRgvfDlbvZ1naYz1x3LrESvqbNsehPREQKWs9Akm88tZVFp0/hHfOnh11OXlLQi0hB+/oTr3GgZ4AvvOu8krx71Fgo6EWkYG3Y08W9v9jOby+ZwyUleC/YsVLQi0hBSqWdLzz8CvWTyviLa88Nu5y8pqAXkYL0/RW7WNfcxf9+13xqq+Jhl5PXFPQiUnBau/u54/FXufysU1i24LSwy8l7owa9md1jZq1mtmFYW72ZPWlmW4PnKUG7mdk3zazJzNab2cJsFi8ipcfd+fzDrzCYTPN/l12gA7BjMJYt+nuB645q+yzwtLvPA54O3gNcD8wLHrcB356YMkVEMn68dg9PbW7lz689hzMaJoddTkEYNejd/X+A9qOalwH3Ba/vA94zrP27nvEiUGdmMyaqWBEpba3d/Xxp+SYWnT5F17M5ASfbRz/d3fcCBM/TgvaZwO5h0zUHbW9gZreZ2UozW9nW1naSZYhIqRjqsulPpLjj/RcRjajLZqwm+mDsSH/yPtKE7n6Xuy9y90UNDQ0TXIaIFJsHV6vL5mSdbNDvH+qSCZ5bg/ZmYPgVhWYBLSdfnogINHf08qXlG1nSWK8um5NwskG/HLg5eH0z8Miw9g8Ho2+WAl1DXTwiIicjnXY+/Z/rcHf+/gMXq8vmJIx60WYz+wFwJTDVzJqBLwJfBR4ws48Au4Abg8kfBd4JNAG9wC1ZqFlESsg9L2znxW3t3HHDRcyurwq7nII0atC7+4eO8dHVI0zrwO3jLUpEBODVfd3c8d9buOa8ady4aFbY5RQsnRkrInmpqzfBH/77Kmor4/zN+y7SiVHjoKAXkbyTSjuf+OEaWjr7+JffXUhDdXnYJRU0Bb2I5J1vPPUaz25p44u/dT6Xnl4fdjkFT0EvInnl0Vf28o8/a+IDi2bxO5fNCbucoqCgF5G8sWpnB5/64VoWzqnjr3TBsgmjoBeRvLDjwGFu/e5KZtRW8G83L6YiHg27pKKhoBeR0LUfHuT3v7MCd+feW5ZQP6ks7JKKyqjj6EVEsunwQJJb7n2Zlq5+fnDrZTROnRR2SUVHW/QiEpqBZIqP/ccqNuzp4p9+e6FG2GSJgl5EQpFKO3/6wDqe23qAr77vQt4+f3rYJRUtBb2I5Fw67XzuofX8dP1evvDO87hx0ezRZ5KTpj56EcmpVNr5ix+t58HVzXz86nnc+rYzwi6p6CnoRSRnkqk0n/7Pdfx4bQufuuZsPnHNvLBLKgkKehHJif5Eik/ev5bHN+7jz689h9t/46ywSyoZCnoRybr2w4N89L6XWbO7k//zm/P5yFt0l6hcUtCLSFbtOHCY3//OCvZ29fPPv72Q6y+cEXZJJUdBLyJZ8/Tm/Xzqh2uJRozv37qUS0+fEnZJJUlBLyITLpV27nzyNb71TBPnn1bDv/zupboNYIgU9CIyofZ29fFnD6zjF68f5IOLZvPlZefrAmUhU9CLyIT5yfoWPv/QKyRSzh03XMQHFutEqHwwrqA3sx3AISAFJN19kZnVAz8EGoEdwAfcvWN8ZYpIPusZSPKXj2zgodV7uHh2Hd/44ALm6uJkeWMituh/w90PDHv/WeBpd/+qmX02eP+ZCfgdEclD65s7+fgP1rCrvZePX3UWf3L1POJRXV0ln2Sj62YZcGXw+j7gWRT0IkVnMJnm7ue38/UntzB1cjk/uHUpl51xSthlyQjGG/QOPGFmDvw/d78LmO7uewHcfa+ZTRtpRjO7DbgNYM4c3RdSpFC4O4++so+vPf4qu9p7ufb86Xzthouoq9LNQvLVeIP+cndvCcL8STN7dawzBiuFuwAWLVrk46xDRHJg1c52/vqnm1mzq5Nzpldz3x8s4YqzG8IuS0YxrqB395bgudXMHgaWAPvNbEawNT8DaJ2AOkUkRDsOHOZrj7/KYxv2Ma26nK/dcCHvv3Q20Yhu3l0ITjrozWwSEHH3Q8HrdwB/BSwHbga+Gjw/MhGFikjubd1/iG8/+zqPrGuhPBbhT99+Nh9961yqyjQyu5CM529rOvCwmQ19z/fd/XEzexl4wMw+AuwCbhx/mSKSK+7Oi9vaueeF7Ty5aT+V8Sg3v6mRj11xBtNqKsIuT07CSQe9u28DLh6h/SBw9XiKEpHc6xlIsnxtC/f9Ygdb9h+irirOx6+ex++/uZH6STrQWsi0/yVSwtydFdvbeWBlM4++spe+RIr5M2q444aLePeC03TpgiKhoBcpQXs6+3hwVTM/WtXMrvZeJpfHeM8lp/H+S2ezcE4dQZesFAkFvUgJSKWddc2dPLuljWe3tLK+uQuAN51xCp+8Zh7XXXCqDrAWsYL+m/3l6wf5uye2UD+pjPqqMuomxakuj1FVFmNSeZSqshiTy2NUlUWJxyLEIkY0YkyrrmDq5DJttUhR29/dz3NbD/Dz19p4bmsbnb0JIgaXzJnCp99xNssWzNSlg0tEQQc9QHkswu72XtY3d9JxOMFgKj2m+SrjUWbXV1Iei5JIpUmmnXg0QlVZlKqyKJXxKBXxKOWxCDWVcRqqy5lWXU5dVZyKWJTyeITKeIwpk+JMqSpTX6aEyt1p7uhjze5OXt7ezi9eP8DrbYcBmDq5nKvPnc4V5zTwtnlTdQZrCSrooH/TmafwpjN//doag8k0fYMpegaT9A4k6RlI0juYIpFKk0o7iVSafV397O7oY3d7L4lUmlg0QjxqJFJO72CSQ/1J2g4NMJBM059I0dWXoHcwddxaKuNRaipj1FbGmVweIxoxIpbZg4hGDDMjHjEaqsuZXlPB9JoKaipjTCrP7HWUxyKUxSKUx6JMqYpTWxnXHoeMaCjUN7Z0sbGlmw17unhlTxcHegYBqCqLsmRuPR9cPJs3nzmV+TNqiOjEppJW0EE/krIgMGur4hP6vT0DSVq7++nqSzCYTNOfTNM7kKSjN0FH7yCdvYN09SXo6kvQM5AknYaUO4PJNGl3nMxKaP2eLg70DOCjXPShLBph6uQyqspjxCKWWa5ohPJ4ZmVQUxFj6uRyplaXUxmPHlmRRSNGbWX8yAqnLBYhHsw3qSzTjTUpWLFoRZL/+hMpmlp72LLvEFv2H2JTSzcbWrro7E0AEI0YZzVM5oqzp7FgTh2XzK7jnFOrdfVI+TVFF/TZMrk8xuSGyRPyXYlUmgM9A/T0Jzk0kOTwQJLBZJrBZJqBZJqDhwdpOzRA26EB+hMpBlOZzxKpNAOJNF19Cba2JoLPx9ZVdbRYxJhckdmbqKnIrBiqK2JUV2Sef32vBGor49RVlTGlquzIiqSmMnM8JB41rTROUDrtdPUl2NvVz77uPvZ29bO/q5/93QPs6+5nf3c/+7r7jwQ6ZDZizp4+mesvOJXzT6vlgpm1nHtqtboNZVQK+hDEoxFm1FZC7fi+x93pCVYSsUiEWNRIpvzX9iwSqcwKoj+RpncwSV8iRU+wcunpT9Ldn6S7L0F3f4IdBw9nVj79SXoGk6PudQyJRozKeJThvQOTy2NMrS5n6uRyYhGjL5GiP5EiYkZdVZy6yjJqq+LUVMSoqcwc9xhaoTm84VjJ0PGSofWJYcSiRjya2dOJx4LXsUimGyw6/j2Woa6+wVQad4Z9LyRSzkAyxUDQVdg7mOLwYJKuvgTdfQk6exPs6+5nb2cfrUE3YDKVWZG3Hx6k/fAgyfSv/wFHLNOfPr2mgllTqljcWM+ptRWcfkoV555aTeMpk4hpS11OgoK+gJkZ1RVv7KKaqG4rd8cdEuk03X1JOnoH6Tg8eGTl0NWXoC+ROhJ06WFrhkP9SQ70DLC/u59U2qmIZ0I7lXZ2HOils6+Tzt4EA8mT2yMZTcQ4soKoiEWoiEdJu5NIOYOpNAZHVgxGppttKNh7BzMrpURq5DWdGWNaCcajxvSaCqZVl1NZFiVeEaMsGmHB7DpOmVzGKZPKObW2glNrK5hRW0HD5HIFuWSFgl6Oycwwg/JIlIbqKA3V5RP+GwPJFN19SQaSqSPHIYzMHsDhwSR9Qej2J9IMJH91QDztkAy2thMpP7LnMtT91TeYOrIX0Z9I059METULjllktvQHk35klFbEIGqZvYLKsiiVZdEjB8jLgvAd2uNIpZ3y4MB5ZvRVZihvVXk006VVEaeuKk59VZkOgkpeUNBLqMpjmZXI0WqZ2IPpIqVM+4kiIkVOQS8iUuQU9CIiRU5BLyJS5BT0IiJFTkEvIlLkFPQiIkVOQS8iUuTMx3pBk2wWYdYG7DzJ2acCByawnEKgZS4NWubSMJ5lPt3dG0abKC+CfjzMbKW7Lwq7jlzSMpcGLXNpyMUyq+tGRKTIKehFRIpcMQT9XWEXEAItc2nQMpeGrC9zwffRi4jI8RXDFr2IiBxHwQa9md1oZhvNLG1mi4767HNm1mRmW8zs2rBqzCYzW2BmL5rZWjNbaWZLwq4pF8zsT4K/141mdkfY9eSKmX3azNzMpoZdS7aZ2d+a2atmtt7MHjazurBrygYzuy74t9xkZp/N5m8VbNADG4D3Af8zvNHM5gM3AecD1wH/bGbFePfkO4Avu/sC4C+D90XNzH4DWAZc5O7nA38Xckk5YWazgbcDu8KuJUeeBC5w94uA14DPhVzPhAsy6Z+A64H5wIeC7MqKgg16d9/s7ltG+GgZcL+7D7j7dqAJKMatXQdqgte1QEuIteTKHwFfdfcBAHdvDbmeXLkT+Asyf+dFz92fcPdk8PZFYFaY9WTJEqDJ3be5+yBwP5nsyoqCDfrjmAnsHva+OWgrNp8E/tbMdpPZsi26rZ4RnA281cxeMrOfm9nisAvKNjN7N7DH3deFXUtI/gB4LOwisiCnOZXX94w1s6eAU0f46Avu/sixZhuhrSC3hI63/MDVwKfc/UEz+wBwN3BNLuvLhlGWOQZMAZYCi4EHzOwML/ChY6Ms8+eBd+S2ouwby/9tM/sCkAS+l8vaciSnOZXXQe/uJxNczcDsYe9nUaDdGsdbfjP7LvCJ4O1/Av+Wk6KybJRl/iPgoSDYV5hZmsx1QtpyVV82HGuZzexCYC6wzswg8295tZktcfd9OSxxwo32f9vMbgZ+E7i60Ffkx5DTnCrGrpvlwE1mVm5mc4FOigCXAAAA7ElEQVR5wIqQa8qGFuCK4PVVwNYQa8mVH5NZVszsbKCMIr4Alru/4u7T3L3R3RvJhMPCQg/50ZjZdcBngHe7e2/Y9WTJy8A8M5trZmVkBpAsz9aP5fUW/fGY2XuBfwQagJ+a2Vp3v9bdN5rZA8AmMrt9t7t7Ksxas+RW4B/MLAb0A7eFXE8u3APcY2YbgEHg5iLd2it13wLKgSeDPZkX3f1j4ZY0sdw9aWZ/DPw3EAXucfeN2fo9nRkrIlLkirHrRkREhlHQi4gUOQW9iEiRU9CLiBQ5Bb2ISJFT0IuIFDkFvYhIkVPQi4gUuf8PBEzldPZLaAMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(b,l)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see on the plot that 0 is really  bad value compared to -3 or -6. Then it seem those give the same range of loss. Now let's pick the bias giving the minimum loss."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-5.454545454545455"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b[l.argmin()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_reg4 = SSD_Head(k,-5.45)\n",
    "models = ConvnetBuilder(f_model, 0, 0, 0, custom_head=head_reg4)\n",
    "learn = ConvLearner(md, models)\n",
    "learn.opt_fn = optim.Adam\n",
    "learn.crit = ssd_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b5354fdc2054cafaf56f69e46a3fcca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 79%|█████████████████████████████████████████████████████████▏              | 50/63 [00:28<00:07,  1.78it/s, loss=107]"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEOCAYAAACaQSCZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4VGX2wPHvSa/UhBYCofcekGYXVNaOiK4i2BDXdW3rrq7rrj/Lqmsva0FwbdixoyCiIIiUgPQAoUpPoSUBUs/vj5mwEROYQGbulPN5nnly585975y54px579tEVTHGGGOOJczpAIwxxgQGSxjGGGM8YgnDGGOMRyxhGGOM8YglDGOMMR6xhGGMMcYjljCMMcZ4xBKGMcYYj1jCMMYY4xFLGMYYYzwS4XQAtSkpKUnT0tKcDsMYYwLGokWLclU12ZNjgyphpKWlkZGR4XQYxhgTMERks6fH2i0pY4wxHrGEYYwxxiOWMIwxxnjEEoYxxhiPWMIwxhjjEUsYxhhjPGIJ4xgOFJeStSufOVm5FBaVOh2OMcY4JqjGYRyP8nJl5fb9bNt7gK17DrJt70G27TnI9n2uv3sOlBw+dszANO6/oIuD0RpjjHNCPmEADH95LsWl5QDERYWTUi+WlPqx9Ghej5T6saTUi+WzJdv5ePFW7j63IzGR4Q5HbIwxvhfyCSMsTJhwdToN4qNIqRdLvbhIROQ3xzVKjOG71dl8tXwHl/Ru7kCkxhjjLGvDAE5pn0zXlLrUj4+qMlkA9G/dgFZJ8by3YIuPozPGGP9gCcNDIsLIvqks2LSbddkFTodjjDE+ZwmjBob3bk5EmPD+wl+cDsUYY3zOEkYNJCdGM6RzYyYv3kZRaZnT4RhjjE9Zwqihy/u1YHdhMdNX7XI6FGOM8SlLGDV0ctskUurFWuO3MSbkWMKoobAwV+P3nHW5/JJ3wOlwjDHGZyxhHIcR6c0JE3g/wxq/jTGhwxLGcWhaN5bTOzTiw4ytlJaV1+q5DxSX8v3qbA6VWKO6Mca/WMI4Tpf3a0F2fhHfrc4+4XOpKsu37uNvnyyn38MzuOb1hYyaOJ99B0uOXdgYY3zEawlDRFJF5HsRyRSRlSJyq3v//SKyTUSWuB/Dqim/SUSWu4/J8Facx+v0Dsk0rhPNewuPv/F738ES3vxpE797bg7nvzCHjxdv5ewuTbh3WCeWbNnLFePnkZNfVHtBG2PMCfDmXFKlwJ2qulhEEoFFIjLd/drTqvqEB+c4XVVzvRfi8YsID2NEn1RenLmOHfsO0rRurEflVJUFG3fz/sItTFm+g6LScro0q8ODF3Xlgh7NqBsbCUC7xgmMe3sRl73yE29d14/m9eO8+XGMMeaYvFbDUNUdqrrYvZ0PZAIp3no/J4zsm0q5wgcLt3p0/Krt+zn7mR8YOX4e01ft4tI+zfnylsFM+dPJjOrf8nCyADitQyPevu4kcguKGPHyTzYdiTHGcT5pwxCRNKAXMN+9648iskxEXhOR+tUUU+AbEVkkImN9EGaNpTaI4+R2SXyQsYWycj3qsV8u287wl+ay72AJj1/anfn3nsnDF3eja0rdasukpzXg/bEDKClTLnvlJ5Zv3VfbH8EYYzzm9YQhIgnAZOA2Vd0PvAS0AXoCO4Anqyk6SFV7A+cCN4vIKdWcf6yIZIhIRk5OTu1/gGO4vG8Ltu09yOysqt+7rFx5bOpq/vjOz3RuVocvbhnMiPRU4qI8uxvYuVkdPhw3gNjIcK54dR4/rc+rzfCNMcZjXk0YIhKJK1lMUtWPAVR1l6qWqWo58CrQr6qyqrrd/Tcb+OQox41X1XRVTU9OTvbGxziqIZ0b0zA+incX/HZMxr6DJVz3xkJemrmeK/ql8s4NJ9EoMabG79EqKZ7JNw2kSd0YRv93Ad/atCTGGAd4s5eUABOBTFV9qtL+ppUOuxhYUUXZeHdDOSISDwyt6jh/EBURxvA+zZmRmU12/qHD+7N25XPRf35kTlYuD1/clUcu6U50xPGv1Nekbgwf3DiATk0SufHtRXzys2ftJsYYU1u8WcMYBIwCzjiiC+2/3d1llwGnA7cDiEgzEfnKXbYxMEdElgILgCmqOtWLsZ6QkX1TKS1XPlrk+hL/ZuVOLn5xLvmHSnh3bH+uPKllrbxPg/goJt3Qn35pDbj9/aV8uWx7rZzXGBO4Xpy5jqtfW+CT9/Jat1pVnQNUtXzdV1Xsq7gFNcy9vQHo4a3Yalub5AT6tWrA+wu3UFKqPP3tWro3r8vLV/WhWT3Putt6KiE6gv9e05erJsznrg+X0Topgc7N6tTqexhjAseanflsyi30yXvZSO9ackW/VDbnHeDpb9dySa8UPrhxQK0niwoxkeG8eFVv6sRGMPatDPYUFnvlfYwx/i+3oIikhCifvJcljFpybtemnNYhmX+e35knL+tBTOTxt1d4olFiDK+MSid7fxF/fHdxrc9pZYwJDHkFxTRMiPbJe1nCqCUxkeG8fk0/rhnUCld7v/f1TK3HQxd35cd1eTz69WqfvKcxxr/kFhT7rIbhzalBjA9clp7Kym37mDBnI11T6nJRr6AaTG+MOYrycmV3YREN462GYTz09/M6069VA/46eRkrttlocGNCxd6DJZQrNLQ2DOOpyPAwXryyNw3joxj7Zga5BTbDrTGhoOL/dWvDMDWSlBDNK6PSySss5uZJiymxRnBjgl5FwrBeUqbGujWvy6PDuzF/424enpLpdDjGGC/LK3B1qU/yUQ3DGr2DzMW9mrNi234mztlIl2Z1GJGe6nRIxhgvyau4JRVvvaTMcbrn3I5k7tjPvZ+uoKxcGdk31WddfY0xvpNXWEyYQL04uyVljlNEeBj/+X1vereox90fL+fKCfPZnOebqQOMMb6TW1BMg/gowsN884PQEkaQqh8fxTvX9+dfF3dj2dZ9nP3MD0yYveGYCz0ZYwKHa1oQ37RfgCWMoBYWJvz+pBZMv+MUBrVJ4qEpmVzy0lzW7Mx3OjRjTC3IKyjy2RgMsIQREprWjWXC6HSevbwnW3Yf4LznZ/PMt2spLrWut8YEsrzCYp+N8gZLGCFDRLiwZwrTbz+FYd2a8sy3WZz//ByWbNnrdGjGmOPkmnjQahjGSxomRPPs5b2YcHU6+w6WMPyluazcbtOJGBNoDpWUUVBUam0YxvvO6tyYr289meiIMF6bs8npcIwxNZTnXgfHV2MwwBJGSKsfH8UlvVP4Ytn2wwOAjDGBITe/YloQq2EYHxk9II3i0nLeW7jF6VCMMTWQV1gx8aDVMIyPtGucyKC2DZk0b7Ot2mdMAMn18TxSYAnD4KplbN93iOmrdjkdijHGQxUTD1oNw/jUmZ0a07x+LK/P3eR0KMYYD+UVFBEbGU5clO+mBLSEYQgPE0b1b8n8jbvJ3LHf6XCMMR7I9fEob7CEYdxG9k0lJjKMN3/a5HQoxhgP5BUW+7T9AixhGLd6cVFc1DOFT37ext4DxU6HY4w5htyCYp+ttFfBEoY5bPTANA6VlPNBhnWxNcbf5RUU+XQeKfBiwhCRVBH5XkQyRWSliNzq3n+/iGwTkSXux7Bqyp8jImtEZJ2I3O2tOM3/dGpah36tGvDmT5trfRr0vQeKWZ9TUKvnNCZUlZcruwt9O48UeHfFvVLgTlVdLCKJwCIRme5+7WlVfaK6giISDvwHGAJsBRaKyOequsqL8RpgzMA0/jBpMd+tzmZI58bHfR5VZfXOfL5bnc33q7NZ/MseyhWGdWvCfed1pmnd2FqM2pjQsv9QCaXlSkMft2F4LWGo6g5gh3s7X0QygRQPi/cD1qnqBgAReQ+4ELCE4WVDOzemad0Y3pi7qcYJo7ColLnr8/hudTYz12SzY98hALo0q8PNp7dFRHhl1npmrsnhT2e249pBrYiKsLuixtRUbkHFtCDBU8M4TETSgF7AfGAQ8EcRuRrIwFUL2XNEkRSg8o30rcBJ3o/URISHcVX/ljw+bQ3rsvNp2yjxmGV+/mUPT3+bxbz1eRSXlRMfFc7J7ZK57axkTuvQiMZ1Yg4fO6JPc/7vi1U8+vVqPlq0lQcu6MLAtkne/EjGBB0nRnmDDxq9RSQBmAzcpqr7gZeANkBPXDWQJ6sqVsW+Km+qi8hYEckQkYycnJxaijq0jeybSlR4GG/M3XzU48rLlZdnrWfEyz+xdmc+Vw9oyTvXn8TP/xjKy6P6MLJvi18lC4DUBnFMGJ3OxNHpFJWW8fsJ87nl3Z/Z6a6NGGOOzYlR3uDlGoaIROJKFpNU9WMAVd1V6fVXgS+rKLoVSK30vDmwvar3UNXxwHiA9PR0W7C6FiQlRHNej6ZMXryVu87pQJ2YyN8ck1tQxB0fLOWHtTkM69aERy7pTt3Y3x5XnTM7NWZQ2yRenrWeF2eu57vMXdw+pD2jB6YRGW63qYw5msMTDwZRLykBJgKZqvpUpf1NKx12MbCiiuILgXYi0kpEooDLgc+9Fav5rTED0zhQXMbkRVt/89rcdbmc++xs5m3I46GLuvKf3/euUbKoEBMZzm1ntWf67afQr1UDHpqSybBnZzNrrdUUjTma3IJiRKB+XM3/vzsR3vwpNwgYBZxxRBfaf4vIchFZBpwO3A4gIs1E5CsAVS0F/ghMAzKBD1R1pRdjNUfo3rwevVrU482fNlPu7mJbWlbOU9+s4cqJ86kTE8FnNw/iqv4tcf02OH4tG8bz2pi+jB/Vh6LScka/toDRry1g7a782vgoxgSdvIIi6sdFEeHj2rg3e0nNoeq2iK+qOX47MKzS86+qO9b4xpiBadz63hJ+yMqhQ5NEbn13CQs27ebSPs154MIutTrpmYgwtEsTTu2QzFs/bebZGVmc88wPXNGvBbcPae/zxj1j/FluQZHPe0iBj3pJmcB0btemPJSYyaNfr2bX/kMUlZbz9MgeXNyrudfeMzoinOtPbs0lvZvz3Iws3pq3mc+WbOfm09tyzaA0YiLDvfbexgSKvIJin7dfgE0NYo4iKiKM3/drweqd+TStG8uXtwz2arKorEF8FPdf0IVpt51C/9YNeGzqas58chZfLN2OqvVtMKEtz4FR3mA1DHMM405tQ+vkeM7u0sSRX/dtGyUwYXRfflyXy0NTMrnl3Z957ceN/OXsjgxo09Dn8RjjD1y3pKyGYfxMbFQ4F/ZMcfxW0KC2SXx5y2D+Pbw7O/Ye4opX5zFq4nyWb93naFzG+FpRaRn5h0ppGO/7GoYlDBMwwsOEy/qmMvOu07h3WCdWbNvH+S/M4aa3F7Eu2yY2NKHhf4P2fF/DsFtSJuDERIZzwymtubxfKhNmb2TC7A1MW7mT4b2bc+tZ7WheP87pEI3xmrzD04JYDcMYjyXGRHL7kPb88JfTuWZQKz5bup0znpjF/Z+vPDw5mzHBJrdilLe1YRhTcw0TornvvM7M/PNpXNI7hbfmbeZ3z81mT6GtHGiCj9UwjKkFzerF8ujw7ky+aSB5BcX883ObHMAEn7wCq2EYU2t6ptbjT2e24/Ol25m6YofT4RhTq/IKi4mOCCM+yvc9Fy1hmKB002lt6JpSh3s/WXH4F5kxwSA33zUG40TncDseljBMUIoMD+OJET3Yf6iEf9itKRNEcguLHWm/AEsYJoh1bFKH285qz5RlO5iyzG5NmeCQV1DkSPsFWMIwQe7GU1rTvXld7vtshXW1NUHBNfGg1TCMqXUR4WE8OaIHBYdKue/TFTZxoQloqkpeodUwjPGado0TuX1Ie75esZMv7NaUCWD7D5VSUqbWhmGMN91wcit6ptbjH5+tIDv/kNPhGHNcKm6rOrWgmCUMExIi3L2mDhSXce8ndmvKBKb/TTxoNQxjvKptowTuGtqB6at28dmS7U6HY0yNHR7l7cBqe2AJw4SYawe3ok/L+vzz85Vk77dbUyaw5BY6N48UWMIwISY8THj80u4cKilj3NuLWPzLHqdDMsZjFTWM+tat1hjfaJ2cwGPDu7M+p5BLXpzL8JfmMnXFTsrKrV3D+LfcgiLqxUUSGe7MV7ctoGRC0kW9UhjSuTEfZmxh4o8bGff2Ilo2jOO6wa24tE9z4qLsfw3jf/IKih3rIQVWwzAhLD46gjGDWjHzz6fz4pW9aRAfxT8+W8nAR7/jiWlrrPut8TtOjvIGSxjGEB4mDOvWlE/+MIjJNw3gpFYN+M/MdQx+9Hse+TrTuuAav5FbWORoDcPq3cZU0qdlA14Z1YBNuYU8910Wr8zaQHREOHcMae90aMa4ahgO9ZACSxjGVCktKZ4nR/QgIkx4bkYWTevGcEW/Fk6HZUJYcWk5+w6WODYGA7x4S0pEUkXkexHJFJGVInLrEa//WURURJKqKV8mIkvcj8+9Facx1RERHr64G6e2T+bvn67gu9W7nA7JhLDdhc6O8gbvtmGUAneqaiegP3CziHQGVzIBhgC/HKX8QVXt6X5c4MU4jalWZHgYL17Zm85N63DzpJ9ZumWv0yGZEOX0PFLgxYShqjtUdbF7Ox/IBFLcLz8N/AWw1kTj9+KjI3htTF+SEqO49vWFbMotdDokE4LyHB7lDT7qJSUiaUAvYL6IXABsU9WlxygWIyIZIjJPRC46yrnHuo/LyMnJqb2gjakkOTGaN67pR7kqY/67wNYJNz53eB6pYKxhVBCRBGAycBuu21T3Av/woGgLVU0Hfg88IyJtqjpIVcerarqqpicnJ9dW2Mb8RuvkBCaM7suOfYe49o0MDhSXOh2SCSFOz1QLXk4YIhKJK1lMUtWPgTZAK2CpiGwCmgOLRaTJkWVVdbv77wZgJq4aijGO6tOyPs9f0YvlW/dyyzs/U1pW7nRIJkTkFhYRFR5GYrRznVu92UtKgIlApqo+BaCqy1W1kaqmqWoasBXorao7jyhbX0Si3dtJwCBglbdiNaYmhnZpwv9d2JUZq7O577OVNrDP+ERufjFJCVG4vlqd4c0axiBgFHBGpe6xw6o7WETSRWSC+2knIENElgLfA4+qqiUM4zdG9W/JH05rw7sLfuHZGVlOh2NCgJNreVfwqG7jHkPxXyAfmIDr9tDdqvpNdWVUdQ5w1FTormVUbGcA17u35wLdPInNGKfcdXYHsvOLeObbLATh1rPaOR2SCWJOj/IGz2sY16rqfmAokAxcAzzqtaiMCQAiwmPDu3Npn+Y8/e1anvxmjd2eMl6TV1Dk6Chv8HxqkIqawjDgv6q6VJy8kWaMnwgPE/49vDsRYcLz362juKycu8/p6Oh9ZhN8VJXcwmJHx2CA5wljkYh8g6uH0z0ikghY9xBjgLAw4V8XdyMiXHhl1gZKSpX7zutkScPUmoKiUopLyx2/JeVpwrgO6AlsUNUDItIA120pYwyupPHghV2JCAvjtR83Ulpezv3ndyEszJKGOXG5BRWjvAPjltQAYImqForIVUBv4FnvhWVM4BER/nl+ZyLDhVdnb6SkTHn4oq6WNMwJ84dR3uB5o/dLwAER6YFrDqjNwJtei8qYACUi/G1Yp8Ndbv86eZmtFW5OWEUNw8nV9sDzGkapqqqIXAg8q6oTRWS0NwMzJlCJCHed3YGI8DCem5FFabny+KXdiQi3BS7N8ckrdH6mWvA8YeSLyD24BuKdLCLhQKT3wjImsIkIdwxpT2SY8OT0tZSWK09d1oNISxrmOFTMI9UgQGoYI3FNAnitqu4UkRbA494Ly5jgcMuZ7YiMCOPRr1dTWlbOs5f3IirCkoapmdyCIurERDj+b8ejd3fP9TQJqCsi5wGHVNXaMIzxwLhT2/D333Xi6xU7+cOkRRSVljkdkgkweQXFJCU6ezsKPEwYInIZsAAYAVyGa12LS70ZmDHB5PqTW/PghV34NjObsW8u4lCJJQ3judyCIpIcHuUNnt+Suhfoq6rZACKSDHwLfOStwIwJNqMGpBEZHsY9nyzn+jcyePXqdGKjwp0OywSAvMJi2jVKcDoMj7vVhlUkC7e8GpQ1xrhd3q8Fj1/ag7nrc7nm9QUUFtkiTObY8gqKHB/lDZ5/6U8VkWkiMkZExgBTgK+8F5YxwevSPs15emRPFm7aw+jXFpB/qMTpkIwfKy0rZ8+BEscnHgTPG73vAsYD3YEewHhV/as3AzMmmF3YM4Xnr+jFki17GTVxAfsOWtIwVdtdWDEtiPM1DI/X+lPVybiWWzXG1IJh3ZoSESbc/M5irpwwj7evO4l6cc5/KRj/4i/zSMExahgiki8i+6t45IvIfl8FaUywGtqlCeNHpbN2VwFj31xk04iY36gY5e30PFJwjIShqomqWqeKR6Kq1vFVkMYEs9M7NuKRi7uxYNNuXvlhvdPhGD9TMco7kBq9jTFedEnvFIZ1a8LT09eyYts+p8MxfiTXPVOtP4zDsIRhjB8QER6+qBv146K4/f0lNrDPHJZXWExkuFAn1uMmZ6+xhGGMn6gfH8UTI3qQlV3AY1NXOx2O8RO5+a61vP1hBUdLGMb4kVPaJzNmYBr//XETs7NynA7H+IG8wmK/aL8ASxjG+J27z+1Im+R4/vzhUvYeKHY6HOMw1yhv59svwBKGMX4nJjKcZy/vRV5BMfd+ugJV62obynILiklyeB2MCpYwjPFDXVPqcvuQ9kxZtoPPlmx3OhzjEFUlr9A/5pECSxjG+K1xp7YhvWV97vtsBdv2HnQ6HOOAwuIyDpWUB/8tKRFJFZHvRSRTRFaKyK1HvP5nEVERSaqm/GgRyXI/bP1wE3LCw4SnR/akvFy584MllNso8JCTV+Afa3lX8GYNoxS4U1U7Af2Bm0WkM7iSCTAE+KWqgiLSAPgncBLQD/iniNT3YqzG+KXUBnH884IuzNuwmwlzNjgdjvGxXD8a5Q1eTBiqukNVF7u384FMIMX98tPAX4DqfjKdDUxX1d2qugeYDpzjrViN8Wcj+jTn7C6NeWLaWjJ32BRuoSTPj0Z5g4/aMEQkDeiFa2nXC4Btqrr0KEVSgC2Vnm/lf8nGmJAiIjxySXfqxEby18nLbILCEJJXGCI1jAoikoBrWvTbcN2muhf4x7GKVbGvyv9LRGSsiGSISEZOjg10MsGpQXwU953XiWVb9/H+wi3HLmCCQkUNo0EodKsVkUhcyWKSqn4MtAFaAUtFZBPQHFgsIk2OKLoVSK30vDlQZd9CVR2vqumqmp6cnFzbH8EYv3FBj2ac1KoB/562mj2FNqAvFMxdn0eTOjHERPrH2u/e7CUlwEQgU1WfAlDV5araSFXTVDUNV2Lorao7jyg+DRgqIvXdjd1D3fuMCVkiwgMXdiX/UCn/nrbG6XCMl2Vs2s3c9XlcN7iV06Ec5s0axiBgFHCGiCxxP4ZVd7CIpIvIBABV3Q08CCx0Px5w7zMmpHVoksg1A9N4b+EvLN2y1+lwjBc9OyOLhvFRXNm/hdOhHObNXlJzVFVUtbuq9nQ/vjrimDRVzXVvZ6jq9ZVee01V27of//VWnMYEmlvPakdSQjT3fbbCGsCD1OJf9jA7K5cbTmlNXJTz05pXsJHexgSYxJhI/v47awAPZs/NyKJ+XCSj+rd0OpRfsYRhTACyBvDgtWTLXmauyeH6k1sTH+0/tQuwhGFMQLIG8OD1/Iws6sVFMnpgmtOh/IYlDGMClDWAB5/lW/cxY3U21w9uRYKf1S7AEoYxAc0awIPLc99lUScmgqv9sHYBljCMCWjWAB48Vm7fx/RVu7hucGvqxEQ6HU6VLGEYE+CsATw4PD9jHYkxEYwZlOZ0KNWyhGFMgLMG8MCXuWM/U1fu5JpBragb65+1C7CEYUxQ6NAkkTHWAB6wXvhuHQnREVzrx7ULsIRhTNC4zd0A/sCXq1C1BvBAsXZXPl+t2MGYgWnUi/OPWWmrYwnDmCCRGBPJHUPas2jzHqatPHI+T+Ovnv9uHXGR4X41yWB1LGEYE0RG9GlOu0YJPDZ1DSVl5U6HY45hXXY+Xy7bztUD06jvJ2teHI0lDGOCSER4GPcM68jG3ELemf+L0+GYY3jhu3XERIRzfQDULsAShjFB5/QOjRjQuiHPzshi/6ESp8Mx1diQU8DnS7dz9YCWNEzwjzW7j8UShjFBRkT427BO7C4s5qWZ650Ox1Tj5VnriYoI4/qTWzsdiscsYRgThLo1r8vFvVJ4bc5Gtu896HQ45gi79h/ik5+3cVl6KsmJgVG7AEsYxgStO4e2R4EnvrHBfP7mvz9uoqxcuX5w4NQuwBKGMUGref04rhmUxic/b2Pl9n1Oh2Pc8g+VMGneZs7t1pQWDeOcDqdGLGEYE8T+cFpb6sVG8shXq20wn594b8EW8otKufGUwKpdgCUMY4Ja3dhIbjmjHXPW5TJrbY7T4YS84tJyJs7ZyIDWDenevJ7T4dSYJQxjgtxV/VvSsmEcj3y12tbMcNgXS7ezc/8hxp4aeLULsIRhTNCLigjjL2d3ZM2ufCYv2up0OCFLVRn/wwY6NE7ktPbJTodzXCxhGBMChnVrQq8W9Xhy+hoOFJc6HU5Imrk2hzW78hl7SmtExOlwjoslDGNCgIhw77BO7NpfxMTZG50OJyS9Mms9TevGcH6PZk6HctwsYRgTItLTGnB2l8a8PGs9OflFTocTUpZu2cu8Dbu5dlAroiIC92s3cCM3xtTYX8/pSFFpOfd8vNwawH1o/A8bSIyJ4PJ+qU6HckK8ljBEJFVEvheRTBFZKSK3uvc/KCLLRGSJiHwjIlXWz0SkzH3MEhH53FtxGhNKWicncN95nfk2cxf3f77Sxmb4wOa8Qr5esYMrT2pJYoz/Lr/qiQgvnrsUuFNVF4tIIrBIRKYDj6vqfQAi8ifgH8C4KsofVNWeXozPmJA0emAa2/ce5JUfNtCsXiw3ndbG6ZCC2oTZG4kIC+MaP19+1RNeSxiqugPY4d7OF5FMIEVVV1U6LB6wnzjG+Nhfz+nIjn2HeGzqaprWjeGiXilOhxSU8gqK+HDRFi7q1YzGdWKcDueEebOGcZiIpAG9gPnu5w8DVwP7gNOrKRYjIhm4aiqPquqn3o/UmNAQFiY8PqI72fmHuOujpTRKjGZg2ySnwwo6b/60mUMl5YwNwGlAquL1Rm8RSQAmA7ep6n4AVb1XVVOBScAfqynaQlXTgd8Dz4hIlfVcBBIxAAASYElEQVRmERkrIhkikpGTY1MfGOOp6IhwXhmVTqukeG58axGrd+53OqSgcrC4jDd/2sRZnRrRtlGi0+HUCq8mDBGJxJUsJqnqx1Uc8g4wvKqyqrrd/XcDMBNXDaWq48ararqqpicnB+boSWOcUjc2ktev6Ud8dARjXlvIjn22dkZt+WjRFvYcKOHGU4OnjcibvaQEmAhkqupTlfa3q3TYBcDqKsrWF5Fo93YSMAhYdeRxxpgT16xeLK9f25fColLGvLaQfQdtWdcTVVauvDp7I71a1CO9ZX2nw6k13qxhDAJGAWdU6h47DHhURFaIyDJgKFDR3TZdRCa4y3YCMkRkKfA9rjYMSxjGeEnHJnV4ZVQfNuQWcONbGRSVljkdUkCbumInv+w+wI2ntAnYaUCqIsHUDzs9PV0zMjKcDsOYgPXpz9u47f0lXNCjGc+M7ElYWPB82fmKqnL+C3M4UFTG9DtOJdzPr6GILHK3Fx+TjfQ2xhx2Ua8U/nJOBz5fup1nZ2Q5HU5AmrMulxXb9nPjqa39PlnUlE+61RpjAsdNp7ZhQ04hz32XRa8W9TitQyOnQwooL89aT+M60UE5tsVqGMaYXxERHrywKx0aJ3Lb+0vYttd6Tnlq6Za9/Lguj+sHtyY6ItzpcGqdJQxjzG/ERoXz0lV9KCtT/jBpsTWCe+jlWeupExPBFSe1cDoUr7CEYYypUqukeB4f0Z2lW/byrymZTofj99bnFDB15U6uHpBGQnRw3u23hGGMqdY5XZtyw8mteOOnzXy+dLvT4fi18bM2EBUexpggmGSwOpYwjDFH9ZdzOtI3rT53T15G1q58p8PxSzv3HeLjn7cysm8qSQnRTofjNZYwjDFHFRkexgu/701cVDg3TVpMYZGtCX6kiXM2UK5ww8nBMclgdSxhGGOOqXGdGJ67ohcbcgq45+PltvBSJfsOlPDO/F84v3tTUhvEOR2OV1nCMMZ4ZGCbJO4c6hrU9/a8zU6H4zfemreJwuIyxoXAQlSWMIwxHrvp1Dac0bERD3y5iiVb9jodjuMOFpfx3x83cUbHRnRsUsfpcLzOEoYxxmNhYcJTl/WgcZ0Ybp60mD2FxU6H5KgPF20hr7CYcUE0hfnRWMIwxtRIvbgoXryyNzn5Rdz32Qqnw3FMSVk5r8zaQJ+W9embFjxTmB+NJQxjTI11b16PP53Zli+X7WDayp1Oh+OIKct2sG3vQW46NbimMD8aSxjGmONy46lt6Ny0Dn//dAX7DoTWokuqysuz1tO+cQJndAydyRktYRhjjktkeBj/vrQ7uwuLeXBKaK1vNnNNDqt35jPu1DYhtWaIJQxjzHHrmlKXm05tw0eLtjJzTbbT4fjMSzPXk1IvlvN7NHM6FJ8KzhmyjDE+c8uZbZm6cid/+3g5024/hcSYSKdDqlUHi8vYvLuQTbkH2JxXyNpdBSzYtJt/nt+ZyPDQ+s1tCcMYc0KiI8L596XdGf7SXB6bupqHLurmdEjH7UBxKe/M/4WsXQVsyitkc94Bdu4/9KtjGsRHcVanxozsm+pQlM6xhGGMOWG9W9TnukGtmDBnI7/r1owBbRo6HdJxefTr1bz502aSEqJJaxjH4HZJpDWMo2XDeNIaxtOiYRx1Y4OrBlUTljCMMbXizqEdmJ65i7s/XsbUW08hNiqwVpzL2pXPpPm/MKp/Sx68qKvT4fil0LoBZ4zxmtiocB4b3p3NeQd48ps1TodTYw9/lUlcVDi3D2nvdCh+yxKGMabW9G/dkKv6t2DijxtZ/Msep8Px2Ky1Ocxck8OfzmhHg/gop8PxW5YwjDG16u5zO9Gsbix/+WhZQKwFXlpWzkNfrqJlwziuHtjS6XD8miUMY0ytSoiO4F+XdGNddgHPz1h3wuf7YW0Of5i0iOdmZLFmZ36tr8Xx7sItZGUXcM+5nYiOCKx2F1+zRm9jTK07tX0yl/Zpzkuz1tOsXiyX9E4hJrJmX8brsgv411eZfLc6m3pxkXy1fCdPTV9LWsM4zu7ShKFdGtMrtf4JjbTed7CEp6ev5aRWDTi7S+PjPk+osIRhjPGK+37Xmaxd+fztk+U8NX0towe05Kr+Lal/jDaCvQeKeebbLN6et5nYyHDuObcjYwalse9ACd+s2sU3q3bx2o8beeWHDSQlRDOkc2PO7tKYgW2SiIqo2U2T/3y/jj0HirnvvM4hM4HgiRBvLbUoIqnAm0AToBwYr6rPisiDwIXufdnAGFXdXkX50cDf3U8fUtU3jvWe6enpmpGRUVsfwRhzglSVuevzGP/DBmatzSEmMozL0lO5bnArWjaM/9WxJWXlvD1vM898m0X+oRIu79eCO4a0Jykh+jfn3XewhJlrsvlm5S5mrsmmsLiMOjGuW2Hndfdsuo7NeYWc9dQsLuqZwuMjetTK5w1EIrJIVdM9OtaLCaMp0FRVF4tIIrAIuAjYqqr73cf8CeisquOOKNsAyADSAXWX7aOqR+12YQnDGP+1Zmc+E2Zv4LMl2ykpL+fszk244ZRW9G5Rn+/XZPPQlEw25BQyuG0Sfz+vk8cr2B0qKWPu+lxe+G4di3/Zyx9Pb8sdQ9of81bVuLcW8UNWDjP/fBqN6sTUxkcMSDVJGF67JaWqO4Ad7u18EckEUlS18rSW8bgSwpHOBqar6m4AEZkOnAO86614jTHe1aFJIo+P6MFdZ3fgjZ828fa8X5i6cicp9WLZtvcgrZPimTg6nTM6NqrR7aGYyHDO6NiYQW2T+MenK3nh+3Ws3rmfp0f2rHZeq3kb8pi6cid3Dmkf0smiprxWw/jVm4ikAT8AXVV1v4g8DFwN7ANOV9WcI47/MxCjqg+5n98HHFTVJ6o491hgLECLFi36bN5si9MbEwgOFJfyYcZWvl6xg6Gdm3BV/5Y1boM4kqry5k+beeDLVbROiufVq9NJS/r1ra/ycuWC/8xhd0Ex3/35tBo3xgebmtQwvN6tVkQSgMnAbRW3olT1XlVNBSYBf6yqWBX7qsxsqjpeVdNVNT05Obm2wjbGeFlcVASjB6bx3tgBXDu41QknCwARYfTANN66th85BUVc+J8fmZ31q9+jTF68lRXb9vOXczqGfLKoKa8mDBGJxJUsJqnqx1Uc8g4wvIr9W4HKU0E2B37TMG6MMVUZ2DaJz28eTJM6MYx+bQETZm9AVSksKuXxaWvokVqPC0JsLYva4LU2DHHdhJwIZKrqU5X2t1PVLPfTC4DVVRSfBvxLRCpWVh8K3OOtWI0xwadFwzg+/sNA7vhgCQ9NySRzRz5JiVFk5xfx0lW9Q2qlvNrizXEYg4BRwHIRWeLe9zfgOhHpgKtb7WZgHICIpAPjVPV6Vd3t7n670F3ugYoGcGOM8VR8dAQvXdmHZ2dk8ewM1+/U87o3pU/LBg5HFph80ujtK9at1hhTnakrdvD63E08eVlPUurFOh2O3/CLbrXGGONPzunalHO6NnU6jIBmkw8aY4zxiCUMY4wxHrGEYYwxxiOWMIwxxnjEEoYxxhiPWMIwxhjjEUsYxhhjPGIJwxhjjEeCaqS3iOTgmm6ksrq4plGv6b4kILdWAzy2quLyZnlPjj/aMTV9za6758fbda/98nbdq46hpap6NtW3qgb1A9fSsDXeB2T4Q6zeLO/J8Uc7pqav2XW3627XPbCveyjckvriBPb52onGUNPynhx/tGNq+ppdd8+Pt+te++XturscdwxBdUuqNolIhno4IZepPXbdnWHX3RmBdt1DoYZxvMY7HUCIsuvuDLvuzgio6241DGOMMR6xGoYxxhiPWMIwxhjjEUsYxhhjPGIJ4ziJSLyILBKR85yOJVSISCcReVlEPhKRm5yOJ1SIyEUi8qqIfCYiQ52OJ1SISGsRmSgiHzkdS4WQSxgi8pqIZIvIiiP2nyMia0RknYjc7cGp/gp84J0og09tXHdVzVTVccBlQMB0RXRSLV33T1X1BmAMMNKL4QaNWrruG1T1Ou9GWjMh10tKRE4BCoA3VbWre184sBYYAmwFFgJXAOHAI0ec4lqgO64h/TFArqp+6ZvoA1dtXHdVzRaRC4C7gRdU9R1fxR+oauu6u8s9CUxS1cU+Cj9g1fJ1/0hVL/VV7EcT4XQAvqaqP4hI2hG7+wHrVHUDgIi8B1yoqo8Av7nlJCKnA/FAZ+CgiHylquVeDTzA1cZ1d5/nc+BzEZkCWMI4hlr69y7Ao8DXliw8U1v/3v1NyCWMaqQAWyo93wqcVN3BqnovgIiMwVXDsGRxfGp03UXkNOASIBr4yquRBbcaXXfgFuAsoK6ItFXVl70ZXBCr6b/3hsDDQC8RucedWBxlCcNFqth3zHt1qvp67YcSUmp03VV1JjDTW8GEkJpe9+eA57wXTsio6XXPA8Z5L5yaC7lG72psBVIrPW8ObHcollBi190Zdt2dEfDX3RKGy0KgnYi0EpEo4HLgc4djCgV23Z1h190ZAX/dQy5hiMi7wE9ABxHZKiLXqWop8EdgGpAJfKCqK52MM9jYdXeGXXdnBOt1D7lutcYYY45PyNUwjDHGHB9LGMYYYzxiCcMYY4xHLGEYY4zxiCUMY4wxHrGEYYwxxiOWMIxjRKTAB+9xgYfT1dfme54mIgOPo1wvEZng3h4jIi/UfnQ1JyJpR07TXcUxySIy1VcxGWdYwjABzz1tdJVU9XNVfdQL73m0edhOA2qcMIC/Ac8fV0AOU9UcYIeIDHI6FuM9ljCMXxCRu0RkoYgsE5H/q7T/U/fKhitFZGyl/QUi8oCIzAcGiMgmEfk/EVksIstFpKP7uMO/1EXkdRF5TkTmisgGEbnUvT9MRF50v8eXIvJVxWtHxDhTRP4lIrOAW0XkfBGZLyI/i8i3ItLYPaX1OOB2EVkiIie7f31Pdn++hVV9qYpIItBdVZdW8VpLEZnhvjYzRKSFe38bEZnnPucDVdXYxLUy5BQRWSoiK0RkpHt/X/d1WCoiC0Qk0V2TmO2+hourqiWJSLiIPF7pv9WNlV7+FLiyyv/AJjioqj3s4cgDKHD/HQqMxzWbZxjwJXCK+7UG7r+xwAqgofu5ApdVOtcm4Bb39h+ACe7tMbgWWwJ4HfjQ/R6dca1NAHAprunSw4AmwB7g0irinQm8WOl5ff43W8L1wJPu7fuBP1c67h1gsHu7BZBZxblPByZXel457i+A0e7ta4FP3dtfAle4t8dVXM8jzjsceLXS87pAFLAB6OveVwfXzNVxQIx7Xzsgw72dBqxwb48F/u7ejgYygFbu5ynAcqf/XdnDew+b3tz4g6Hux8/u5wm4vrB+AP4kIhe796e69+cBZcDkI87zsfvvIlzrZlTlU3WtX7JKRBq79w0GPnTv3yki3x8l1vcrbTcH3heRpri+hDdWU+YsoLPI4dmt64hIoqrmVzqmKZBTTfkBlT7PW8C/K+2/yL39DvBEFWWXA0+IyGPAl6o6W0S6ATtUdSGAqu4HV20EeEFEeuK6vu2rON9QoHulGlhdXP9NNgLZQLNqPoMJApYwjD8Q4BFVfeVXO10LJp0FDFDVAyIyE9eyuACHVLXsiPMUuf+WUf2/7aJK23LEX08UVtp+HnhKVT93x3p/NWXCcH2Gg0c570H+99mOxeMJ4FR1rYj0AYYBj4jIN7huHVV1jtuBXUAPd8yHqjhGcNXkplXxWgyuz2GClLVhGH8wDbhWRBIARCRFRBrh+vW6x50sOgL9vfT+c4Dh7raMxrgarT1RF9jm3h5daX8+kFjp+Te4ZikFwP0L/kiZQNtq3mcurqmwwdVGMMe9PQ/XLScqvf4rItIMOKCqb+OqgfQGVgPNRKSv+5hEdyN+XVw1j3JgFK61po80DbhJRCLdZdu7aybgqpEctTeVCWyWMIzjVPUbXLdUfhKR5cBHuL5wpwIRIrIMeBDXF6Q3TMa1uM0K4BVgPrDPg3L3Ax+KyGwgt9L+L4CLKxq9gT8B6e5G4lVUsYqaqq7GtQRq4pGvuctf474Oo4Bb3ftvA+4QkQW4bmlVFXM3YIGILAHuBR5S1WJgJPC8iCwFpuOqHbwIjBaRebi+/AurON8EYBWw2N3V9hX+V5s7HZhSRRkTJGx6c2MAEUlQ1QJxraO8ABikqjt9HMPtQL6qTvDw+DjgoKqqiFyOqwH8Qq8GefR4fgAuVNU9TsVgvMvaMIxx+VJE6uFqvH7Q18nC7SVgRA2O74OrkVqAvbh6UDlCRJJxtedYsghiVsMwxhjjEWvDMMYY4xFLGMYYYzxiCcMYY4xHLGEYY4zxiCUMY4wxHrGEYYwxxiP/D4O2RbNqXJ8nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2fafd5358677466f9fa76216d2cc7f3d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      19.675239  18.063049 \n",
      "    1      17.461465  16.142759                                                                                        \n",
      "    2      16.149953  15.330717                                                                                        \n",
      "    3      15.024208  14.744383                                                                                        \n",
      "    4      14.235962  14.417577                                                                                        \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[14.417577]"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lr, 1, cycle_len=5, use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the same ballpark as -6, which was expected, but not as good."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The next thing I tried was to compute the loss over the entire validation set, and not just a mini-batch, and find the bias that minimizes it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_loss1(min_b=-10,max_b=0, n=50):\n",
    "    test_biases = np.linspace(min_b,max_b,n)\n",
    "    losses = np.zeros(n)\n",
    "    nb = learn.model[8].out.oconv1.bias.size(0)\n",
    "    fp = FloatProgress(min=0,max=n)\n",
    "    display(fp)\n",
    "    for i in range(n):\n",
    "        learn.model[8].out.oconv1.bias = torch.nn.Parameter(torch.Tensor([test_biases[i]]*nb).cuda())\n",
    "        for data in md.val_dl:\n",
    "            x,y=data\n",
    "            x,y = V(x),V(y)\n",
    "            z = learn.model(x)\n",
    "            losses[i] += ssd_loss(z,y)\n",
    "        fp.value+=1\n",
    "    return test_biases,losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_reg4 = SSD_Head(k,0)\n",
    "models = ConvnetBuilder(f_model, 0, 0, 0, custom_head=head_reg4)\n",
    "learn = ConvLearner(md, models)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It takes longer to run, so let's focus near the values we know are good."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fdadc81600f543b482ba7a1209160fee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>FloatProgress</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "FloatProgress(value=0.0, max=20.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "b1,l1 = test_loss1(-7,-4,20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-5.105263157894737"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd8FWX6///XlZ5QEhISIAVCk95DEbCBHxtVASsKKh9suPzWVVZ01+7quvqR1V1ZFUEEFRVRELsgrqCUhJLQjRAgAZJAIARSSLm/f2TYXxYDOakz55zr+XjkwTlz5iTv28GLyXVm7luMMSillPJcPnYHUEopVb+00CullIfTQq+UUh5OC71SSnk4LfRKKeXhtNArpZSH00KvlFIeTgu9Ukp5OC30Sinl4fzsDgDQvHlzEx8fb3cMpZRyK0lJSUeMMZFV7eeIQh8fH09iYqLdMZRSyq2IyD5X9tPWjVJKeTiXC72I+IrIJhFZbj0XEXlWRHaLyA4R+V2F7a+ISKqIJItI3/oKr5RSqmrVad1MB3YATa3nk4E4oLMxpkxEoqztVwMdra+BwGzrT6WUUjZw6YxeRGKBEcCcCpvvAZ4yxpQBGGOyrO1jgHdMubVAmIi0qsPMSimlqsHV1s0sYAZQVmFbe+AGEUkUkS9FpKO1PQY4UGG/dGubUkopG1RZ6EVkJJBljEk666VAoNAYkwC8Ccw985ZKvs1vVjcRkanWPxKJ2dnZ1YytlFLKVa6c0Q8BRotIGrAIGCYiCyk/U//Y2ucToKf1OJ3y3v0ZscDBs7+pMeYNY0yCMSYhMrLKy0CVUkrVUJWF3hgz0xgTa4yJB24EVhpjJgKfAsOs3S4BdluPlwG3WVffDAJyjTGH6j467Mk+yV+/2okuh6iUUudWmxumngfeFZHfAyeBKdb2L4BrgFQgH7i9VgnPY8WOLGav+pWIRgFMuahdff0YpZRya9Uq9MaYVcAq6/Fxyq/EOXsfA9xXB9mqNOWitiTuy+G5L3fSLTqUC9tHNMSPVUopt+LWd8aKCC9O6EWbiBDuf38jh3IL7I6klFKO49aFHqBJkD+vT+xH/ulS7n13I6dLyqp+k1JKeRG3L/QAHVs04W/je7Fp/3GeXr7d7jhKKeUoHlHoAUb0bMXUi9uxYO0+Pk5KtzuOUko5hscUeoAZV3ZiULtwHvkkha0ZuXbHUUopR/CoQu/n68M/bu5Ls5AA7nk3ieP5p+2OpJRStvOoQg/QvHEgr03sy+HcQv6/DzZTVqY3UymlvJvHFXqAvq2b8diobqzalc2sFb/YHUcppWzlkYUeYOLA1ozrG8srK35hxY5Mu+MopZRtPLbQiwjPXtudrq2a8vsPNrPv6Cm7IymllC08ttADBPn78vqt/RAR7lqQRMHpUrsjKaVUg/PoQg8QFx7C32/sza7MPB75JEVnulRKeR2PL/QAl3aK4veXX8AnmzJ45+d9dsdRSqkG5RWFHmDaZR0Y3jmKp5dvJzEtx+44SinVYLym0Pv4CP93Q29imgVz77sbycortDuSUko1CK8p9AChwf78a2I/ThQWM+3dTRSX6kyXSinP51WFHqBLq6Y8f11P1qfl8NwXO+2Oo5RS9c7rCj3A2D4xTB4cz9w1e1m25TfrliullEfxykIP8Mg1XUho04w/Lk5m/9F8u+MopVS98dpCH+Dnwys39cHXR/T6eqWUR/PaQg8QHRbMH6/qxOrUI3y8McPuOEopVS+8utAD3DKwDf3aNOPp5dvJziuyO45SStU5ry/0Pj7CX8f1oOB0KU/perNKKQ/k9YUeoENUE6YN68BnWw7qlMZKKY+jhd5y9yXtuaBFY/706VbyCovtjqOUUnXG5UIvIr4isklEllvP3xaRvSKy2frqbW0XEXlFRFJFJFlE+tZX+LoU4OfD8+N6cvhEIX/7epfdcZRSqs5U54x+OrDjrG0PGWN6W1+brW1XAx2tr6nA7NrHbBh9Wzdj0oXxLFi7Tyc+U0p5DJcKvYjEAiOAOS7sPgZ4x5RbC4SJSKtaZGxQD13ZiejQYB5ekkJRiS5UopRyf66e0c8CZgBnzwL2rNWeeVlEAq1tMcCBCvukW9vcQqNAP565tjupWSd57ftf7Y6jlFK1VmWhF5GRQJYxJumsl2YCnYH+QDjwxzNvqeTb/Oa2UxGZKiKJIpKYnZ1dvdT17LJOUYztHc1rq1LZnZlndxyllKoVV87ohwCjRSQNWAQME5GFxphDVnumCJgHDLD2TwfiKrw/FvjNzGHGmDeMMQnGmITIyMhaDaI+/HlkVxoH+jFjcTKlZTo9glLKfVVZ6I0xM40xscaYeOBGYKUxZuKZvruICDAW2Gq9ZRlwm3X1zSAg1xhzqH7i15+IxoE8Nqormw8cZ8HPaXbHUUqpGqvNdfTvikgKkAI0B56xtn8B7AFSgTeBe2uV0EZje8dw8QWRvPD1LjKOF9gdRymlakScMGtjQkKCSUxMtDtGpdKP5XPFy/9mYNtw5k7uT/kvMEopZT8RSTLGJFS1n94ZW4XYZiE8eEUnvt+VrYuUKKXckhZ6F0waHE+vuDCe/Gw7OadO2x1HKaWqRQu9C3ytGS5PFBTzjM5wqZRyM1roXdS5ZVPuubQ9SzZl8MNuZ133r5RS56OFvhqmDetA+8hGPLIkhVNFJXbHUUopl2ihr4ZAP1+eH9eTjOMF/N+3u+2Oo5RSLtFCX03948OZOKg189bsZfOB43bHUUqpKmmhr4EZV3UmqkkQD3+cTHHp2fO8KaWUs2ihr4GmQf48PbY7Ow/n8foPOsOlUsrZtNDX0P90bcGInq14ZUUqyenawlFKOZcW+lp4ekx3IpsEcveCJI6eLLI7jlJKVUoLfS2ENwrg9Vv7cfTUaaa9t4kS7dcrpRxIC30tdY8J5S/X9uDnPUd5/suddsdRSrmJsjLDE8u2saUBrt7TQl8HxvWLZfLgeOas3svSzRl2x1FKuYGN+4/x9k9ppB09Ve8/Swt9HXl0RBcGxIfzx4+T2X7whN1xlFIO93nKIQL8fBjepUW9/ywt9HXE39eHf97Sl7DgAO5amMjxfJ3lUilVubIywxcph7jkgkgaB/rV+8/TQl+HIpsEMntiXzJzi7j//U261qxSqlIb9x8j80QRI3u2apCfp4W+jvVp3YynxnTjx1+O8NI3u+yOo5RyoIZs2wDU/+8MXujGAa1JzsjltVW/0j0mlGt6NMy/2kop52votg3oGX29eXxUV/q0DuPBj7awOzPP7jhKKYdo6LYNaKGvN4F+vvxrYj8aBfpx14IkcguK7Y6klHKAhm7bgBb6etWiaRCv3dKXAzn5PPDBZsr0w1mlvJodbRvQQl/v+seH8/iorqzYmcXfV/xidxyllI3saNuAFvoGMXFQG8b3i+XvK37h2+2ZdsdRStnEjrYNaKFvECLCM2O70zM2lAc+2Myv2SftjqSUamB2tW2gGoVeRHxFZJOILD9r+6sicrLC80AR+UBEUkVknYjE111c9xXk78vsif3w9/PhrgVJnNTFxZXyKna1baB6Z/TTgR0VN4hIAhB21n53AseMMR2Al4G/1iqhB4kJC+YfN/dh75FT/OFD/XBWKW9iV9sGXCz0IhILjADmVNjmC/wNmHHW7mOA+dbjxcBwEZHaR/UMg9s3Z+bVnfl6WyazdRlCpbyCnW0bcP2MfhblBb3iyhrTgGXGmENn7RsDHAAwxpQAuUBELXN6lDuHtmVM72he/GYX3+/KsjuOUqqe2dm2ARcKvYiMBLKMMUkVtkUDE4BXK3tLJdt+06MQkakikigiidnZ2dWI7P5EhOev60nnlk2Z/v4mDuUW2B1JKVWP7GzbgGtn9EOA0SKSBiwChgHbgA5AqrU9RERSrf3TgTgAEfEDQoGcs7+pMeYNY0yCMSYhMjKytuNwO8EBvsy+pS+nS8t49JOtGKP9eqU80Zm2zaU2tW3AhUJvjJlpjIk1xsQDNwIrjTHNjDEtjTHx1vZ868NXgGXAJOvxeGt/rWKViG/eiIeu7MzKnVks3XzQ7jhKqXpwpm0zwqa2DdTPdfRvARHWGf4DwMP18DM8xuTB8fRtHcYTn20jO6/I7jhKqTpmd9sGqlnojTGrjDEjK9neuMLjQmPMBGNMB2PMAGPMnroI6ql8fYQXxvckv6iUJ5ZtszuOUqoOOaFtA3pnrCN0iGrC9Ms78nnKIb7aevZFTEopd+WEtg1ooXeMqRe3o1t0U/706TZdb1YpD+GEtg1ooXcMf18fXhjfk+P5p3lq+Xa74yilaskpbRvQQu8o3aJDuefS9izZmKE3Uinl5pzStgEt9I4zbVgHOkY15pElKeQV6qpUSrkrp7RtQAu94wT6+fLC+J5knijk+S932h1HKVUDTmrbgBZ6R+rTuhl3Dm3Lu+v28/OvR+2Oo5SqJie1bUALvWM98D+diI8I4Y8fJ5N/WueuV8qdOKltA1roHSs4wJfnx/Vkf04+L32z2+44SikXOa1tA1roHW1QuwgmDmrN3DV72bj/mN1xlFIucFrbBrTQO97DV3chOjSYGYuTKSoptTuOUqoKTmvbgBZ6x2sc6MdfrutBatZJXl2RWvUblFK2cWLbBrTQu4VLLohkfL9YZv/wK1szcu2Oo5Q6Bye2bUALvdv484iuhDcKYMbiZIpLy6p+g1KqwTmxbQNa6N1GaIg/z4ztzvZDJ3hdFxVXynGc2rYBLfRu5cpuLRnRsxWvrEjll8w8u+MopSpwatsGtNC7nSdHd6NRoC8PLU6mtExXaFTKKZzatgEt9G6neeNAnhjdjc0HjjNvzV674yilcHbbBrTQu6XRvaK5vEsUL36zi7Qjp+yOo5TXc3LbBrTQuyUR4ZmxPfD38eGPHydTpi0cpWzl5LYNaKF3Wy1Dg/jzyK6s25vDa6v0Riql7OL0tg1ooXdrExJiGdM7mv/7djdrUo/YHUcpr+T0tg1ooXdrIsJfru1Bu8jG/O79TRzOLbQ7klJex+ltG9BC7/YaBfrxr4l9KSguZdp7G/WuWaUakDu0bUALvUfoENWE58f1JHHfMf6qyw8q1WDcoW0DWug9xuhe0Uy6sA1zVu/lq62H7I6jlFdwh7YNVKPQi4iviGwSkeXW87dEZIuIJIvIYhFpbG0PFJEPRCRVRNaJSHz9RFdne2REF3rFhfHQR8ns1evrlapX7tK2geqd0U8HdlR4/ntjTC9jTE9gPzDN2n4ncMwY0wF4GfhrnSRVVQr08+W1W/ri6yvcszCJgtO6UIlS9eWHX7LJPFHEqF7RdkepkkuFXkRigRHAnDPbjDEnrNcECAbO3LUzBphvPV4MDLf2UQ0gJiyYWTf0ZldmHn9euhVj9GYqperD3NV7iWoSyJXdWtodpUquntHPAmYA/3VJh4jMAw4DnYFXrc0xwAEAY0wJkAtEnP0NRWSqiCSKSGJ2dnbN0qtKXdopivuHdWRxUjofJh6wO45SHmd3Zh4//nKESYPjCfBz/kedVSYUkZFAljEm6ezXjDG3A9GUt3RuOPOWSr7Nb04rjTFvGGMSjDEJkZGR1UutqjR9eEcu6ticPy/dpqtSKVXH5q3ZS6CfDzcNaG13FJe48k/REGC0iKQBi4BhIrLwzIvGmFLgA2CctSkdiAMQET8gFMipw8zKBb4+wqwbehMeEsC9724kt6DY7khKeYScU6dZsjGD6/rGEt4owO44Lqmy0BtjZhpjYo0x8cCNwErgVhHpAP/p0Y8CzlzAvQyYZD0eD6w02ii2RUTjQP55S18OHi/gwY+2aL9eqTrw3rp9FJWUcceQeLujuKymzSUB5otICpACtAKesl57C4gQkVTgAeDhWqdUNdavTTMeuaYL327P5I1/77E7jlJu7XRJGe/8vI+LL4ikY4smdsdxWbUu/jTGrAJWWU+HnGOfQmBCrVKpOnX7kHiS9h3jha930TsujIHtfvPZuFLKBZ+nHCQrr4gXxsfbHaVanP9xsao1EeH5cT1oEx7CtPc3kZWnk58pVV3GGN5avZf2kY24uKN7XUCihd5LNAny57WJfckrLOb+9zZRopOfKVUtifuOsTXjBHcMbYuPj3vdGqSF3ot0btmUv1zbg3V7c3jp2912x1HKrbz1417CQvy5rk+s3VGqTQu9l7mubyw3DWjN7FW/8t32TLvjKOUWDuTk8832w9w8oDXBAb52x6k2LfRe6PFRXeke05QHPtzM/qP5dsdRyvHe/ikNHxFuuzDe7ig1ooXeCwX5+zL7ln4A3PteEoXFOvmZUueSV1jMBxsOMKJnK1qGBtkdp0a00HupuPAQXr6hN9sOnmD6ok2UlunNVEpV5qPEdE4WlXDHkLZ2R6kxLfRebHiXFvx5RFe+3pbJs5/vqPoNSnmZ0jLD2z+lkdCmGb3iwuyOU2POni1f1bs7hrYl/VgBc9fsJbZZMHcMdd+zFqXq2nc7Mtmfk8/DV3e2O0qtaKFXPDqiCxnH83n68+1EhwVzVXfnz6+tVEOYu3ovMWHBXNHV2UsFVkVbN8qa6bIPvePCmL5oExv3H7M7klK225qRy7q9OUweHI+fr3uXSvdOr+pMcIAvc25LoGVoEFPmJ7LvqK45q7zb3DV7CQnw5fr+cXZHqTUt9Oo/IhoHMm9yf4wxTJ63gZxTp+2OpJQtsvIK+WzLQa5PiCM02N/uOLWmhV79l3aRjZkzKYGM4wX87zuJeo298koLf95HSZlh8uB4u6PUCS306jf6tQln1g292bj/GA98uJkyvcZeeZHC4lIWrtvP8M4tiG/eyO44dUILvarUNT1a8eg1Xfgi5TDPfanX2CvvsXRzBjmnTnPH0Hi7o9QZvbxSndOdQ9tyICefN3/cS2yzECZ5yK+xSp2LMYa5q9Po0qopF3rQAj16Rq/OSUR4bFQ3Lu/Sgic/28a3Otul8nBrUo+yKzOPO4bEU74ctmfQQq/Oy9dHePWmPvSICeX+9zey5cBxuyMpVW/mrtlL88YBjOoVbXeUOqWFXlUpOMCXOZP6E9kkkDvnb9CpjZVH2pN9kpU7s5g4qA1B/u435/z5aKFXLolsEsjbtw+guNQw+e31HM/Xa+yVZ5m3Jo0AXx9uGdjG7ih1Tgu9cln7yMa8eVsC6Tl6jb3yLLn5xSxOSmdM72gimwTaHafOaaFX1TKgbTgvXd+LDWnHePCjLXqNvfII72/YT0FxKbe78Zzz56OXV6pqG9UrmozjBTz/5U5imgUz8+oudkdSqsaKS8uY/1Mag9tH0DW6qd1x6oUWelUjd13cjvRj+bz+wx7CQwK465L2dkdSqka+2nqYQ7mFPD2mu91R6o3LrRsR8RWRTSKy3Hr+rojsEpGtIjJXRPyt7SIir4hIqogki0jf+gqv7CMiPDm6OyN7tuK5L3fy9pq9dkdSqkbmrtlLfEQIwzpH2R2l3lSnRz8dqHgv/LtAZ6AHEAxMsbZfDXS0vqYCs2sfUzmRr4/w8g29uaJrC574bDvvr99vdySlqmXj/mNs2n+c24e0xcfHc26QOptLhV5EYoERwJwz24wxXxgLsB6ItV4aA7xjvbQWCBORVnWcWzmEv68Pr97ch0s7RfLIJyks2ZhudySlXDZ39V6aBPkxvl9s1Tu7MVfP6GcBM4Cys1+wWja3Al9Zm2KAAxV2Sbe2nf2+qSKSKCKJ2dnZ1QqtnCXQz5d/TezHhe0iePCjLSxPPmh3JKWqlJp1ki+3HuamAa1pFOjZH1dWWehFZCSQZYxJOscurwH/Nsb8eOYtlezzm2vwjDFvGGMSjDEJkZGRLgdWzhTk78ucSQn0a9OM6Ys28822w3ZHUuqcSssMMxZvoXGgH1Mu8sxLKity5Yx+CDBaRNKARcAwEVkIICKPA5HAAxX2Twcqrr0VC+gpnhcICfBj7uT+9IgJ5b73NvL9riy7IylVqbd/SmPj/uM8PqorUU2C7I5T76os9MaYmcaYWGNMPHAjsNIYM1FEpgBXAjcZYyq2dJYBt1lX3wwCco0xh+ojvHKeJkH+zL99ABe0aMLdC5L4KfWI3ZGU+i9pR07xt693MqxzFNf2+U1X2SPV5s7YfwEtgJ9FZLOIPGZt/wLYA6QCbwL31i6icjehIf4suHMg8RGNuHN+IhvScuyOpBQAZWWGGR8n4+/jw1+u7eFRUxGfT7UKvTFmlTFmpPXYzxjT3hjT2/p6ytpujDH3Wa/1MMYk1kdw5WzhjQJYOGUgrcKCuH3eBjbr9MbKAd5dt4/1e3P408gutAz1/JbNGTrXjao3kU0CeW/KIMIbBXDbW+vYmpFrdyTlxQ7k5PPclzu5qGNzrk+Iq/oNHkQLvapXLUODeO9/B9IkyJ9b31rHrsN5dkdSXsgYw8wlKQjw/LieXtOyOUMLvap3sc1CeHfKQPx9fbhlzjp+zT5pdyTlZT7YcIDVqUeYeU0XYsKC7Y7T4LTQqwYR37wR7/3vIMBwy5vrdJUq1WAO5Rbw7Oc7GNQunJsHtLY7ji200KsG0yGqMQunDKSwpJSb3lxLxvECuyMpD2eM4ZElKZSUGV4Y18uj57M5Hy30qkF1btmUBXcM5ERhMTe/uZbME4V2R1IebMnGDL7flc2MqzrROiLE7ji20UKvGlyP2FDm3zGAI3lF3PzmWnJO6fqzqu5lnSjkyc+2kdCmGZMujLc7jq200Ctb9G3djLmT+3PgWAFTdf1ZVceMMTz66VaKSsp4YXxPr23ZnKGFXtlmYLsIXprQi8R9x5ixOJnyGa+Vqr3Pkg/x7fZM/nDFBbSLbGx3HNt59tycyvFG9Ypmf04+f/t6F20iQvjDFZ3sjqTc3JGTRTy+dCu94sK4c2g7u+M4ghZ6Zbt7L23P/qP5vLoyldbhIUzwsrsWVd16fNk2ThWV8uL4nvh6ecvmDG3dKNuJCM9c250hHSKYuSRFZ7xUNfbV1kN8nnyI6Zd3pGOLJnbHcQwt9MoR/H19eO2WfrRt3oi7FyaRmqVTJajqOXbqNH/6dCvdY5oy9WJt2VSkhV45RmiwP3Mn9yfAz5fb397AkZNFdkdSbuSp5ds5nl/MC+N64e+rpa0i/a+hHCUuPIQ5kxLIzitiyny97FK5ZsWOTD7ZlMF9l3Wga3RTu+M4jhZ65Ti948KYdUMftqQf54EPN1NWppddqnPLLSjmkU9S6NyyCfdd1sHuOI6khV450lXdW/LoNV34IuUwf/16p91xlIM9+/l2jpw8zd/G9yLAT0taZfTySuVYdw5tS9rRU7z+wx7ahDfi5oHeOfOgOrcfdmfzYWI6917anh6xoXbHcSwt9MqxRIQnRnUj/VgBf166ldhmwVx8QaTdsZRD5BYUM/PjZDpENeZ3wzvaHcfR9Pcc5Wh+vj784+a+dIxqzL3vbmTn4RN2R1IOUFZmeOCDzWTlFfHihF4E+fvaHcnRtNArx2sc6Me82/vTKNCXO+ZtIEunNvZ6r65MZcXOLB4f1ZXecWF2x3E8LfTKLbQKDeatSf05XlDMnfMTyT9dYnckZZPvd2Yxa8VuxvWNZeKgNnbHcQta6JXb6B4Tyqs39WHbwVymL9pMqV526XX2HT3F9EWb6NqqKc9e293rFvmuKS30yq0M79KCx0d149vtmTz7+Q6746gGlH+6hLsWJOHjI/xrYj/ty1eDXnWj3M6kwfGkHT3F3DV7aRMRwqTB8XZHUvXMGMPMJSnsysxj/u0DiAv33mUBa8LlM3oR8RWRTSKy3Ho+TURSRcSISPMK+4mIvGK9liwifesjuPJufxrRlcu7tODJz7bxxLJt5BYU2x1J1aN5a9JYuvkgD17RSS+xrYHqtG6mAxV/V14DXA7sO2u/q4GO1tdUYHZtAipVGV8f4ZWbenPzwNbM/zmN4S+tYnFSuk6X4IHW7TnKs1/s4IquLbj30vZ2x3FLLhV6EYkFRgBzzmwzxmwyxqRVsvsY4B1Tbi0QJiKt6iKsUhWFBPjxzNgefDZtKHHhITz40RYmvP4z2w7m2h1N1ZHDuYXc994m2oSH8NL1vfTD1xpy9Yx+FjADKHNh3xjgQIXn6dY2pepF95hQPr57MC+M70nakVOMenU1jy3dSm6+tnPcWVFJKfe8m0TB6RJev7UfTYL87Y7ktqos9CIyEsgyxiS5+D0r+yf3N79Pi8hUEUkUkcTs7GwXv7VSlfPxEa5PiGPlHy7l1kFtWLh2H8NeWsWHGw5oO8dNPb18O5v2H+fFCb10tahacuWMfggwWkTSgEXAMBFZeJ7904GKi37GAgfP3skY84YxJsEYkxAZqR+uqLoRGuLPk2O689n9Q2nbvBEzPk7mutk/kZKu7Rx38lHiARau3c9dl7Tj6h7a+a2tKgu9MWamMSbWGBMP3AisNMZMPM9blgG3WVffDAJyjTGH6iauUq7pFh3KR3dfyEsTepF+rIDR/1zNo5+kcDz/tN3RVBW2ZuTy6KdbGdIhgoeu6GR3HI9Q4xumROR3IpJO+Rl7soic+aD2C2APkAq8Cdxb65RK1YCIMK5fLCsfvITJg+NZtOEAl724ivfX79d2jkPlnDrNXQuSiGwcyCs39sFPlwSsE2KM/X/hExISTGJiot0xlIfbefgEj326jfVpOfSKDeWpMd3ppRNiOUZpmWHS3PWsT8th8d0X0jNWj01VRCTJGJNQ1X76z6XyGp1bNuWDuwYx64beHMwtZOxra5i5JJmTRTpBmhO8+M0uVqce4Zkx3bXI1zEt9MqriAhj+8Sw8g+XMGVoWz5MTGfcaz+x/2i+3dG82ldbDzF71a/cPLA11/ePq/oNqlq00Cuv1CTIn0dHdGX+7QM4fKKQ0f9czU+pR+yO5ZVSs07yhw+30DsujMdHdbU7jkfSQq+82tCOzVl63xAiGwdy69z1zP8pDSd8buUt8gqLuWtBIsEBvsye2JdAP52Rsj5ooVdeL755I5bcO5jLOkXy+LJtzFySwukSV24CV7VRWmZ48KMtpB3N5x8396VVaLDdkTyWFnqlKG/lvHFrAtMu68CiDQe4Zc5ajpwssjuWxzqUW8DNb67l622ZzLy6M4PaRdgdyaNpoVfK4uMjPHhlJ169qQ8pGbmMfnU1WzP0jtq69s22w1z99x9JycjlxQm9uHNoW7sjeTwt9EqdZVRKL1jnAAAKV0lEQVSvaBbfPRiA8f/6ic+2/GYGD1UDhcWlPLZ0K1MXJBETFszy+4cyvl+szkjZALTQK1WJ7jGhLJ02lO7Rodz//iZe/HqX3k1bC79k5jH2n2t45+d9TBnaliX3DqZdZGO7Y3kNLfRKnUNkk0De+99B3Ng/jn98n8rUBUnkFerUx9VhjOH99fsZ9Y/VZOcVMW9yf/40sqteXdPAtNArdR4Bfj48d10Pnhzdje93ZXHdaz+x7+gpu2O5hdz8Yu57byMzl6SQ0CacL6dfxGWdo+yO5ZW00CtVBRFh0uB4FtwxgOyTRYz+xxrW6M1V55W0L4drXvmRb7Zl8vDVnXnnjgFENQ2yO5bX0kKvlIsGd2jOsvuG0qJpILfNXc+8NXv15qqzlJYZXl3xC9e/vhZfH2HxPYO5+5L2+PjoB6520kKvVDW0jghhyb1DGNY5iic/286Mxclknii0O5YjnLk2/qVvdzOiRys+/91QeuvsoI7gZ3cApdxN40A/Xp/Yj1nf7eaVlal8vDGdwe2bM7ZPDFd1b0njQO/73+rb7Zk8tHgLp0vKeHFCL8b1jdHLJh1E56NXqhb2ZJ/k080H+XRTBvtz8gny9+F/urbk2j7RXNQxEn8PXzijsLiU577Ywfyf99Etuimv3tRHL5tsQK7OR6+FXqk6YIxh4/7jfLopg+XJBzmWX0x4owBG9WzF2D4x9I4Lc5sz3KKSUvIKSzhRUMyJ//xZzImCEuvP///51oO57Mk+xZShbXnoqk562WQD00KvlE1Ol5Tx793ZfLI5g++2Z1JUUkabiBDG9o5hbJ8Y2jZvZHdEikvL+HrbYZZuPsiRk0WcKCguL+6FxRQWn39CNz8fITTYn6bB/oQ3CmDaZR30skmbaKFXygHyCov5authPt2cwU+/HsUY6B0XxrV9YhjZsxURjQMbNM/Rk0Us2nCABT/v4/CJQmLCgmkX2YimQf40Dfaz/vSnaZCf9efZ2/0J8vdxm99OPJ0WeqUc5nBuIcu2ZPDJpoPsOHQCXx/h4o7NubJbS4Z1iSKqSf1dZ77tYC5vr0lj6ZaDnC4p46KOzZk8OJ5LO0Xhq5c+ui0t9Eo52M7DJ/h000E+23KQjOMFAPSKDWV4lxYM7xJF11ZNa33WXFJaxjfbM3l7TRrr03II9vdlXL8YJl0YT8cWTepiGMpmWuiVcgPGGHYezmPFjky+25HFlvTjGAPRoUEM6xLF8C4tuLBdBEH+rn/ImXPqNIs27Gfhz/s4mFtIXHgwky6MZ0K/OEJD/OtxNKqhaaFXyg1l5xXx/c4svtuRyY+/HKGguJSQAF8u6tic4V1aMKxzFM3P0dfffvAE839K49PNGRSVlDGkQwSTB7dlWGdtz3gqLfRKubnC4lJ+3nOUFTsyWbEji0O5hYiUf5h7udXi6RDZmO92ZDJvTRrr9uYQ5O/DdX1jmXRhPJ1aanvG02mhV8qDGGPYfugEK3ZksWJHJlvSy1e+CvL3obC4jNhm5e2Z6xO0PeNNXC30Lt+rLSK+QCKQYYwZKSJtgUVAOLARuNUYc1pEAoF3gH7AUeAGY0xaDcaglLKICN2iQ+kWHcrvhnck60QhK3eW9/Qv7RTF5V1aaHtGnVN17s+eDuyo8PyvwMvGmI7AMeBOa/udwDFjTAfgZWs/pVQdimoaxI0DWvPcdT25sltLLfLqvFwq9CISC4wA5ljPBRgGLLZ2mQ+MtR6PsZ5jvT5c9O4KpZSyjatn9LOAGcCZe6MjgOPGmBLreToQYz2OAQ4AWK/nWvsrpZSyQZWFXkRGAlnGmKSKmyvZ1bjwWsXvO1VEEkUkMTs726WwSimlqs+VM/ohwGgRSaP8w9dhlJ/hh4nImQ9zY4GD1uN0IA7Aej0UyDn7mxpj3jDGJBhjEiIjI2s1CKWUUudWZaE3xsw0xsQaY+KBG4GVxphbgO+B8dZuk4Cl1uNl1nOs11caJ1zDqZRSXqo2qyL8EXhARFIp78G/ZW1/C4iwtj8APFy7iEoppWqjWmueGWNWAausx3uAAZXsUwhMqINsSiml6oBnr3OmlFLKGVMgiEg2sK+Gb28OHKnDOHbSsTiPp4wDdCxOVZuxtDHGVHk1iyMKfW2ISKIrcz24Ax2L83jKOEDH4lQNMRZt3SillIfTQq+UUh7OEwr9G3YHqEM6FufxlHGAjsWp6n0sbt+jV0opdX6ecEavlFLqPNyu0IvIByKy2fpKE5HN59jvKhHZJSKpIuLYu3NF5H4r5zYReeEc+6SJSIo1ZscuxeXiWBx9XETkCRHJqPB37Jpz7Of4Y1KNsTj6mFQkIg+KiBGR5ud4vbTCeJc1dL7qcGEsk0TkF+trUmX7uMwY47ZfwEvAY5Vs9wV+BdoBAcAWoKvdeSvJeRnwHRBoPY86x35pQHO789Z2LO5wXIAngAdd2M8djkmVY3GHY1IhaxzwNeX33FT63x44aXfOuhgL5Sv37bH+bGY9blbTn+d2Z/RnWIuZXA+8X8nLA4BUY8weY8xpymfdHNOQ+Vx0D/C8MaYIwBiTZXOe2nBlLO5yXLyJOx2TlylfF8MTPlisaixXAt8aY3KMMceAb4GravrD3LbQAxcBmcaYXyp57T+Ln1gqLoziJBcAF4nIOhH5QUT6n2M/A3wjIkkiMrUB81WHK2Nxl+MyTUSSRWSuiDQ7xz7ucEyg6rG4xTERkdGUr1e9pYpdg6x1LtaKyNgq9rWFi2Op0+NSrUnNGoqIfAe0rOSlR40xZ6ZDvonKz+bBxcVPGsL5xkL5f/9mwCCgP/ChiLQz1u9uFQwxxhwUkSjgWxHZaYz5d70Gr0QdjMURx6WKccwGnqY819OUtwfvqGRfdzgmrozFEccEqhzLI8AVLnyb1tZxaQesFJEUY8yvdZnTFXUwljo9Lo4s9MaYy8/3urWgyXVAv3Ps8p/FTywVF0ZpUOcbi4jcAyyxiuF6ESmjfN6L/1pyyxhz0PozS0Q+ofzX7QYvKnUwFkccl6r+fp0hIm8Cy8/xPRx/TCo6z1gccUzg3GMRkR5AW2CLtfx0LLBRRAYYYw6f9T3OHJc9IrIK6EP5ZxANqg7Gkg5cWuF5LNbMwTXhrq2by4Gdxpj0c7y+AegoIm1FJIDyBVOc+An8p5Sv2IWIXED5h2H/NbmRiDQSkSZnHlN+JrC1gXO6osqx4AbHRURaVXh6LZX8t3aXY+LKWHCDY2KMSTHGRBlj4k35AkjpQN+zi7yINBORQOtxc8pXx9ve4IHPw9WxUP5B7RXWmJpR/nfs65r+XHct9DdyVttGRKJF5Av4z6Lk0yj/D7MD+NAYs63BU1ZtLtBORLZS/iHYJGOMqTgWoAWwWkS2AOuBz40xX9mU93yqHIubHJcXrMsmkym/kuj38N9/v3CfY1LlWNzkmJyTiCSIyBzraRcg0Tou31N+cYCjCv35VByLMSaH8nbbBuvrKWtbzb73b9vBSimlPIm7ntErpZRykRZ6pZTycFrolVLKw2mhV0opD6eFXimlPJwWeqWU8nBa6JVSysNpoVdKKQ/3/wChOQ6hKoiD/gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(b1,l1)\n",
    "b1[(l1).argmin()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The minimum is slightly different, and just for the sake of it, let's try with this value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_reg4 = SSD_Head(k,-5.1)\n",
    "models = ConvnetBuilder(f_model, 0, 0, 0, custom_head=head_reg4)\n",
    "learn = ConvLearner(md, models)\n",
    "learn.opt_fn = optim.Adam\n",
    "learn.crit = ssd_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8c246550a7649f5b97cec0462306160",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 79%|█████████████████████████████████████████████████████████▏              | 50/63 [00:30<00:07,  1.65it/s, loss=105]"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEOCAYAAACaQSCZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4VGXax/HvnU4KoYWeEHrvoSiogIqKil2xgg0RUXR1911Xd+27uruiK4qKYMcuIiArFkABFQi9Q+gh9NATQsr9/jGDGzGBCcyZM5ncn+uaKzNnznPml0PIneeU5xFVxRhjjDmZMLcDGGOMKR+sYBhjjPGJFQxjjDE+sYJhjDHGJ1YwjDHG+MQKhjHGGJ9YwTDGGOMTKxjGGGN8YgXDGGOMT6xgGGOM8UmE2wH8qUaNGpqamup2DGOMKTfmz5+/W1WTfFk3pApGamoq6enpbscwxphyQ0Q2+bquHZIyxhjjEysYxhhjfGIFwxhjjE+sYBhjjPGJFQxjjDE+sYJhjDHGJ1YwgoyqsiLrAAWFRW5HMcaY37CCEWQ+nLuFfi/N5OHxS7H51o0xwcQKRhBZuHkvj01cRs2EaD6dn8lbsze6HckYY35lBSNI7D6Ux93vL6B2YgxT7z+bC1rX4umvVvDjml1uRzPGGMDBgiEiySIyXURWishyERnuXf64iGwVkUXeR79S2m8UkaXedUJ6vI+CwiKGfbCAvTlHefXGzlSNi2LEtR1oViuBYR8sYMPuw25HNMYYR3sYBcCDqtoS6A7cIyKtvO+9oKodvI8pJ9hGb+86aQ7mdN0/p67ml/XZ/P2KtrSplwhAXHQEb9ySRkR4GHe8M48DR/JdTmmMqegcKxiquk1VF3ifHwRWAvWc+rzyavKSLEb/uJ6buzfgqs71f/NecrVYRt3YiU17chj+4UIKi+wkuDHGPQE5hyEiqUBHYI530TARWSIib4pI1VKaKfCNiMwXkcEBiBlwa3Yc5E+fLaFTShX+ekmrEtfp3qg6j/dvzfTVu/jn1FUBTmiMMf/jeMEQkXjgc+B+VT0AvAo0BjoA24DnS2naQ1U7ARfhOZx1dinbHywi6SKSvmtX+TlBfOBIPkPem09sVDijbuxMVETp/xQ3dW/ATd1TeP2H9XyxMDOAKY0x5n8cLRgiEomnWIxT1fEAqrpDVQtVtQh4A+haUltVzfJ+3Ql8cYL1RqtqmqqmJSX5NAeI64qKlIc+Wcym7BxevqETtRNjTtrmsUtb071RNf7v86Us2rIvACmNMea3nLxKSoCxwEpVHVFseZ1iq10BLCuhbZyIJBx7DvQtab3y6tUf1vHNih38pV9Lujeq7lObyPAwRt3YmZoJ0Qx+N50dB444nNIYY37LyR5GD+BmoM9xl9D+03u57BKgN/AAgIjUFZFjV0zVAmaJyGJgLvCVqn7tYNaA+XHNLv79zWoubV+X23qklqlttbgoxgxM41BeAYPfTedIfqEzIY0xpgQSSsNPpKWlaTBP0Zp9+CjnPj+DmgkxfHHPmcRGndoMuV8v286Q9+fzpwubM7RXEz+nNMZUJCIy39dbF+xO7wAa/eN69uXm89L1HU+5WABc2KY2vZsn8foP6+3+DGNMwFjBCJDdh/J456eN9G9fl+a1E057e384vzn7c/N5c9YGP6QzxpiTs4IRIKN/XE9eQSH3ndvUL9trWz+RC1rXYuzMDezLOeqXbRpjzIlYwQiAnQeP8O7PG7m8Qz0aJ8X7bbsPnN+MQ0cLGP3jer9t0xhjSmMFIwBem7Ge/ELlXj/1Lo5pUbsyl7Sry9s/bWT3oTy/btsYY45nBcNhOw4c4f05m7iyYz0a1ojz+/bvP68pR/ILeW3GOr9v2xhjirOC4bBR0zMoKlLu7ePf3sUxjZPiuaJjfd77ZZPdzGeMcZQVDAdt25/Lh3O3cHXn+qRUj3Xsc4af25TCIuWV6RmOfYYxxljBcNAr0zNQlHt6O3tzXUr1WK5JS+bDuZvJ3Jvj6GcZYyouKxgOydybw8fztnBtWjLJ1ZzrXRxzb58mCMLL06yXYYxxhhUMh7wyPQNBHO9dHFO3SiVu6JbCp/Mz2WhTuhpjHGAFwwFbsnP4ND2T67smU7dKpYB97tDejYkMF/7z/dqAfaYxpuKwguGAkdPWEhYmDA1Q7+KYmgkxDDwjlQmLtrJ2x8GAfrYxJvRZwfCzjbsP8/mCrdzYLYValU8+MZK/3XVOY2Ijw3nxO+tlGGP8ywqGn700bS2R4cLdvRq78vnV4qK4rWdDvlq6jRVZB1zJYIwJTVYw/Gj9rkNMWLiVm7s3oGZC4HsXx9xxViMqx0Qw4ts1rmUwxoQeKxh+9NL3a4mOCOeuc9zpXRyTWCmSO89qxHcrdzBr7W5XsxhjQocVDD9QVd74cT0TFmUx8MxUasRHux2J289qSJOa8Qz/aCFZ+3LdjmOMCQFWME5TYZHy5OQVPDNlJRe3rcP95zkzZlRZxUZF8NpNnckrKOLucQvIK7D5v40xp8cKxmk4kl/IvR8u4K3ZG7mtR0NGXt+RmMhwt2P9qknNeP59TTsWb9nHE5NWuB3HGFPOWcE4RftyjnLL2LlMWbqdRy9uyd8ubUVYmLgd63cubFOHu85pxAdzNvNJ+ha34xhjyrEItwOUR1v35TLwzbls3pPDyOs7cmn7um5HOqE/9m3O0sz9PDphGa3qVKZNvUS3IxljyiHrYZTRiqwDXPHKbHYcOMI7t3UN+mIBEBEexsjrO1I9Loq73pvP3sM2B7gxpuysYJTB7IzdXPv6z4SHCZ8NOZMzGld3O5LPqsdH8+pNndl1MI/hHy+isEjdjmSMKWfskBQwYeFWilQRgTDxnIcIEyFMxLsMMvfm8tzXq2hUI563b+tCncTADSroLx2Sq/B4/9b85YulvPjdGh7s29ztSMaYcsQKBvDn8Us4kl900vW6N6rG6zenkVgpMgCpnHF912QWbt7LyGkZtKtfhfNb1XI7kjGmnBDV0Dk0kZaWpunp6WVutyU7hyJVitRzE57naM2x1/za+2haM4HwILwSqqyO5Bdy9Ws/sWl3DhPv7UnDGnFuRzLGuERE5qtqmi/rOnYOQ0SSRWS6iKwUkeUiMty7/HER2Soii7yPfqW0v1BEVotIhoj82amcAMnVYmlQPY6GNeJolBRPk5rxNKmZQLNaCTSvnUDLOpVpUbtySBQLgJjIcF69sTPh4cKQ9+aTc7TA7UjGmHLAyZPeBcCDqtoS6A7cIyKtvO+9oKodvI8pxzcUkXDgFeAioBVwfbG2xg+Sq8XynwEdWbPzIH8Zv5TT7WmGUk/VGFMyxwqGqm5T1QXe5weBlUA9H5t3BTJUdb2qHgU+Ai5zJmnFdU6zJB44rxkTFmXx/i+bTnk7Y2auJ+3p71i4ea8f0xljgk1ALqsVkVSgIzDHu2iYiCwRkTdFpGoJTeoBxW9LzsT3YmPKYFjvJvRunsSTk1ew4BR+4X+xMJOnv1rJ/tx87ngnnU17bD5xY0KV4wVDROKBz4H7VfUA8CrQGOgAbAOeL6lZCctKPOYhIoNFJF1E0nft2uWn1BVHWJjwwnUdqFU5hnvGLWDPoTyf285cu4s/frqEMxpVZ/J9PSlUZdBb88i2GwONCUmOFgwRicRTLMap6ngAVd2hqoWqWgS8gefw0/EygeRir+sDWSV9hqqOVtU0VU1LSkry7zdQQVSJjeK1mzqz5/BR7vtooU839S3bup8h782nSc14Xr+lMy1qV2bMLWls3ZfLne+mcyTfRsc1JtQ4eZWUAGOBlao6otjyOsVWuwJYVkLzeUBTEWkoIlHAAGCiU1kNtKmXyNOXtWF2xh5GfLv6hOtuyc5h0FvzqBIbxdu3dqVyjOe+lLTUarx4XQcWbN7LHz5ZRJHdTW5MSHGyh9EDuBnoc9wltP8UkaUisgToDTwAICJ1RWQKgKoWAMOAqXhOln+iqssdzGqAa7skM6BLMq9MX8d3K3aUuE724aMMfHMu+YVFvHNbF2on/nYq2n5t6/BIv5ZMWbqdv09ZGYjYxpgAcexOb1WdRcnnIn53Ga13/SygX7HXU0pb1zjn8f6tWZa1nwc+WcTke3vSoPr/burLPVrI7e/MY+u+XMbd0Y0mNRNK3MbtPRuSuTeXMbM2UK9qJW7t0TBQ8Y0xDrLBB81vHLupL0yEIe8v+PVcREFhEcM+WMDiLfv4z4COpKVWK3UbIsJfL2lF31a1eHLyCqYu3x6o+MYYB1nBML+TXC2WFwd0YNX2Azw6YRmqyl+/XMb3q3byxGVtuLBN7ZNuIzxM+M+AjrSvX4X7Plx4SpfsGmOCixUMU6LezWtyb5+mfDY/k5vGzuHDuVsY1rsJN3dv4PM2KkWFM3ZgGrUTY+weDWNCgBUMU6rh5zbl7GZJzM7Yw9Wd6/Ng32Zl3kb1+GjevrUrqsqtb83jaMHJRwU2xgQnKximVOFhwsjrO/L8Ne35x5VtETm1wRcb1ojj2avasX73YX5cYzdXGlNeWcEwJ5RYKZKrOtcnMvz0flT6tKhJldhIJi0p8f5LY0w5YAXDBERkeBgXtanDtyt2kHvU7gI3pjyygmECpn/7uuQcLeT7VSXfFGiMCW5WMEzAdG1YjZoJ0UxcZIeljCmPrGCYgAkPEy5pV5cZq3dx4Ei+23GMMWVkBcME1KXt63C0sIipy+zub2PKGysYJqA6JFchuVolJi3Z5nYUY0wZWcEwASUiXNquLrMzdpdpsiZjjPusYJiA69+hLoVFyhQ7LGVMuWIFwwRc81oJNK0ZzyS7WsqYcsUKhgk4EeHS9nWZuzGbbftz3Y5jjPGRFQzjikvb1wVg8mI7+W1MeWEFw7iiYY042tZLtLGljClHrGAY1/RvX5clmfvZsNvmyTCmPLCCYVxzcbs6AExebL0MY8oDKxjGNXWrVKJrajU7LGVMOWEFw7jq0vZ1WLPjEKu2H3A7ijHmJKxgGFdd1LYO4WHCJDssZUzQs4JhXFUjPpozG1dn0uJtqKrbcYwxJ2AFw7iuf/u6bM7OYXHmfrejGGNOwAqGcV3f1rWJCg+ziZWMCXKOFQwRSRaR6SKyUkSWi8jw495/SERURGqU0r5QRBZ5HxOdymncl1gpknOaJzF5SRaFRXZYyphg5WQPowB4UFVbAt2Be0SkFXiKCXA+sPkE7XNVtYP30d/BnCYI9G9fl50H85i7IdvtKMaYUjhWMFR1m6ou8D4/CKwE6nnffgH4E2B/ThoAzm1Zk0qR4XZPhjFBLCDnMEQkFegIzBGR/sBWVV18kmYxIpIuIr+IyOVOZzTuio2K4PxWtZi0OIsZq3e6HccYUwLHC4aIxAOfA/fjOUz1CPA3H5qmqGoacAPwoog0LmX7g72FJX3Xrl3+im1ccN+5TUhKiGbQW/O454MF7DhwxO1IxphixMlr30UkEpgMTFXVESLSFvgeyPGuUh/IArqqaqnTr4nI28BkVf3sRJ+Xlpam6enpfslu3JFXUMjrP6zn5ekZRIeH8dAFzbmpewPCw8TtaMaEJBGZ7/3j/KScvEpKgLHASlUdAaCqS1W1pqqmqmoqkAl0Or5YiEhVEYn2Pq8B9ABWOJXVBI/oiHDuO7cp39x/Nh1SqvDYxOVcMWo2y7baPRrGuM3JQ1I9gJuBPsUuj+1X2soikiYiY7wvWwLpIrIYmA48q6pWMCqQ1BpxvHtbV166viNZ+47Q/+VZPDFpOYfyCtyOZkyF5eghqUCzQ1KhaX9uPv+auopxczZTKyGGpy5vw/mtarkdy5iQEBSHpIzxl8RKkTx9eVvG330mVeOiuOu9dOZttPs1jAk0Kxim3OiYUpVPh5xB/aqx3P/RIg4cyXc7kjGuKyxSNu0JzKyVVjBMuRIfHcF/BnRg+4Ej/HXCMrfjGOOawiJl4uIsLnjxRwaM/oW8gkLHP9MKhil3OqZU5f5zm/Lloiy+WJjpdhxjAqqwSPly0Vb6vvAD9324EAEeubglkWHO/zqPcPwTjHHA0N5N+HHtLv46YTmdU6qRUj3W7UjGOKqwSJm8JIuXvl/Lul2HaVYrnldu6MRFbWoTFqD7lKyHYcql8DDhhes6IAL3f7yQgsIityMZ44jiPYrhHy0iIiyMV27oxNfDz+bidnUCVizACoYpx+pXjeWZK9qyYPM+Rk7LcDuOMX63YPNezi9WKEbd2In/Dj8r4IXiGDskZcq1/u3rMmP1TkZOW8tZTWuQllrN7UjG+MX+3HyGvr+A8DDh1Rs7cUHrwB16Ko31MEy590T/1tSvGstwu9TWhJAnJi1n16E8Xr2pExe1dadHcTyfCoaIDBeRyuIxVkQWiEhfp8MZ44uEmEhe9F5q+ze71NaEgG+Wb2f8gq3c07sJ7epXcTvOr3ztYdymqgeAvkAScCvwrGOpjCmjTilVGX5uUyYsymLCwq1uxzHmlGUfPspfvlhK67qVGda7idtxfsPXgnGsL9QPeMs7+ZH7/SNjirmndxO6pFbl0QnL2JKdc/IGxgQZVeXRCUvZn5vP89e2JyoiuM4a+Jpmvoh8g6dgTBWRBMCuYzRBJTxMGHFtBwQYOm4BOw/aBEymfJm0ZBtTlm7ngfOb0aJ2Zbfj/I6vBeN24M9AF1XNASLxHJYyJqgkV4vlhes6sHbnQS55aRbpNkihKSd2eoe76ZhShcFnNXI7Tol8LRhnAKtVdZ+I3AQ8CtiMNiYondeqFl8M7UFsVDgDRv/Cm7M2EErD+JvQo6o8PH4peQWFPH9NeyLCg+tQ1DG+pnoVyBGR9sCfgE3Au46lMuY0taxTmS+H9aR3i5o8OXkF9364kMM2+ZIJUp/Oz+T7VTv50wUtaJQU73acUvlaMArU8yfaZcB/VPU/QIJzsYw5fYmVInn9ps786cLmTFm6jctfmU3GzkNuxzLmNzL35vDkpBV0a1iNQWemuh3nhHwtGAdF5GE8U65+JSLheM5jGBPUwsKEob2a8N7t3cg+fJTLXp7FlKXb3I5lDABFRcr/fb4EVeXf17QPipvzTsTXgnEdkIfnfoztQD3gX46lMsbPejSpweT7etK0VgJDxy3gma9W2ICFxnXj5mxidsYeHr2kFcnVgn/EZZ8KhrdIjAMSReQS4Iiq2jkMU67USazEx3d15+buDXhj5gauGPUTExdnkW+Fw7hg4+7D/H3KKs5plsSALslux/GJr0ODXAvMBa4BrgXmiMjVTgYzxgnREeE8dXkb/jOgA/tz87nvw4X0fG4aL32/ll0H89yOZyqQ575eRXiY8OxVbREJ7kNRx/g6Wu0jeO7B2AkgIknAd8BnTgUzxkmXdajHpe3qMmPNTt6avZER367h5WkZXNKuDgPPTKV9cvCM32NCz+rtB/nvsu3c26cJdRIruR3HZ74WjLBjxcJrDzbSrSnnwsKEPi1q0adFLTJ2HuLdnzfy+fxMxi/cSseUKgw6M5WL2tQJuuEZTPn38vQM4qLCua1HQ7ejlImv/xO+FpGpIjJIRAYBXwFTnItlTGA1qRnPk5e14ee/nMtjl7Zi7+GjDP9oEf1fnkX24aNuxzMhJGPnISYvyeLmM1KpGhfldpwy8fWk9x+B0UA7oD0wWlX/z8lgxrihckwkt/ZoyLQHezHqxk5s2H2Ym8fOYX+uzbNh/GPU9AxiIsK546zy1buAMsy4p6qfA587mMWYoBEWJvRrW4fYqHDufDedQW/N5b3buxEfbZNUmlO3cfdhvlycxa1nplIjPtrtOGV2wh6GiBwUkQMlPA6KyIGTtE0WkekislJElovI8OPef0hEVERqlNJ+oIis9T4Glv1bM+b09Wpek5HXd2JJ5n7ueGceR/IL3Y5kyrFRMzKICBMGnx2cgwuezAkLhqomqGrlEh4JqnqysXcLgAdVtSXQHbhHRFqBp5gA5wObS2ooItWAx4BuQFfgMRGpWrZvzRj/uLBNbUZc2545G7K567355BVY0TBltyU7h/ELtnJ91xRqVo5xO84pcezyD1XdpqoLvM8PAivx3CEO8AKeQQxLG0L0AuBbVc1W1b3At8CFTmU15mQu61CPZ69syw9rdnHvBwvtZj9TZq/+sI4wEe46p3z2LiBAl8aKSCrQEc8Nf/2Brd5Z+0pTD9hS7HUm/ys2xrjiui4pPH5pK75ZsYMHP1lMYZENmW58k7Uvl0/Tt3Btl/rl6r6L4zl+Bk9E4vGcLL8fz2GqR/DMDX7CZiUsK/F/p4gMBgYDpKSknHpQY3wwqEdDcvOLeO7rVVSKDOcfV7YN+gHjjPte/2EdqjDknMZuRzktjvYwRCQST7EYp6rjgcZAQ2CxiGwE6gMLRKT2cU0zgeKDq9QHskr6DFUdrappqpqWlJTk72/BmN+5u1dj7uvThI/Tt/DEpOU2OZM5oZ0HjvDhvC1c3bk+9asG/wCDJ+JYD0M8g6OMBVaq6ggAVV0K1Cy2zkYgTVV3H9d8KvD3Yie6+wIPO5XVmLJ64Pxm5OYX8sbMDVSJjeKB85u5HckEqdd/XE9hkTK0VxO3o5w2J3sYPfDMn9FHRBZ5H/1KW1lE0kRkDICqZgNPAfO8jye9y4wJCiLCX/q15LIOdRk1I4PMvTluRzJBaPehPMbN2cTlHeqRUr189y7A2aukZqmqqGo7Ve3gfUw5bp3UY70LVU1X1TuKvfemqjbxPt5yKqcxp0pE+L8LWyAIL0/LcDuOCUJvzFzP0YIi7uldvs9dHGOjqhlzGupWqcQN3VL4dH4mG3cfdjuOCSLZh4/y3s+buLR93aCep7ssrGAYc5qG9mpMRJjw0vdr3Y5igsibszaQm1/IsN7l/9zFMVYwjDlNNSvHcMsZDZiwaCsZOw+5HccEgf05+bzz00b6talD01oJbsfxGysYxvjBkHMaExMZzovfrXE7igkCr/6wjoN5BQzrEzq9C7CCYYxfVI+P5tYeqUxeso1V2084LqcJcWt3HGTMzPVc3bk+LeucbMi98sUKhjF+cudZjUiIjuCFb62XUVGpKo9OWEZcdAQPX9TC7Th+ZwXDGD+pEhvF7Wc1ZOryHSzbut/tOMYFXyzcypwN2fz5ohZUL4fzXZyMFQxj/Oi2ng1JrBTJCOtlVDj7c/J55quVdEypwnVpySdvUA5ZwTDGjyrHRDL47EZMW7WTBZv3uh3HBNC/vlnF3pyjPH15m5AdkNIKhjF+NujMVKrHRdm5jApk0ZZ9jJuzmUFnNqR13US34zjGCoYxfhYXHcGQcxozc+1u5m6wIdBCXWGR8uiEpdRMiOaB85u6HcdRVjCMccBN3RuQlBDN89+stuHPQ9z7v2xi2dYD/PWSViTERLodx1FWMIxxQKWocO7p1Zg5G7L5ad0et+MYh+w8cIR/T13NWU1rcHHbOm7HcZwVDGMcMqBrCnUSY6yXEcKe/moleYVFPHVZGzxTAIU2KxjGOCQmMpxhfZqwYPM+Zqze5XYc42ezM3YzcXEWd5/TmNQacW7HCQgrGMY46JrOyaRUi+XeDxfyyvQMjuQXuh3J+EFeQSF/nbCM1Oqx3N0rNOa68IUVDGMcFBURxnu3d+WMxtX519TVnPv8D0xYuJWiIjtEVZ6N/mE963cf5snL2hATGe52nICxgmGMwxpUj+ONW9L48M7uVI2L5P6PF3H5qNl2yW05tXlPDi9Pz+DitnU4u1mS23ECygqGMQFyRuPqTLynJyOubc+ug3lc+/rP3PVeOhtspr5y5dmvVxIeJvz1klZuRwk4KxjGBFBYmHBlp/pMe7AXD/Vtxsy1uzl/xA88MWk5+3Py3Y5nTiJ9YzZTlm5nyDmNqZ0Y43acgLOCYYwLKkWFM6xPU2b8sRfXpCXzzk8buev9dLv8NoipKk9/tZJalaO546yGbsdxhRUMY1xUMyGGf1zZlscubc0v67OZuXa325FMKSYv2caiLft4qG9zYqMi3I7jCisYxgSB67umUL9qJf5tN/kFpbyCQp77ehUt61Tmyk713Y7jGisYxgSBqIgwhp/blCWZ+5m6fIfbccxx3vlpI5l7c3n04paEh+jQ5b6wgmFMkLiiYz0aJ8Ux4tvVFNp9GkFj7+GjjJyWQe/mSfRoUsPtOK6ygmFMkIgID+PBvs1Zs+MQExdvdTuO8Xpp2loO5xXwcL+WbkdxnRUMY4LIha1r07puZV74di35hUVux6nwNuw+zHs/b2JA1xSa1UpwO47rHCsYIpIsItNFZKWILBeR4d7lT4nIEhFZJCLfiEjdUtoXetdZJCITncppTDAJCxMeuqA5m7Nz+CR9i9txKrzn/ruK6IgwHjivmdtRgoKTPYwC4EFVbQl0B+4RkVbAv1S1nap2ACYDfyulfa6qdvA++juY05ig0qtZEmkNqvLS92ttsEIXzd2QzdfLt3N3r8YkJUS7HScoOFYwVHWbqi7wPj8IrATqqeqBYqvFAXZ2z5hiRIQ/XtCcHQfyeO/nTW7HqZCKipRnvlpB7cox3N6zkdtxgkZAzmGISCrQEZjjff2MiGwBbqT0HkaMiKSLyC8icvkJtj3Yu176rl0254AJDd0aVefsZkmMmpHBwSM2ZEigTVqSxeLM/Tx0QXMqRVWc0WhPxvGCISLxwOfA/cd6F6r6iKomA+OAYaU0TVHVNOAG4EURKXHQeVUdrappqpqWlFSxRo40oe2hvs3Ym5PPm7M2uh2lQjmSX8g/v15NqzqVubJjPbfjBBVHC4aIROIpFuNUdXwJq3wAXFVSW1XN8n5dD8zA00MxpsJoV78KF7auzZiZ69l7+KjbcSqMt3/ayNZ9npv0wirwTXolcfIqKQHGAitVdUSx5U2LrdYfWFVC26oiEu19XgPoAaxwKqsxweoPfZtx6GgBr/24zu0oFcKug3m8Mi2Dc1vU5MwKfpNeSZzsYfQAbgb6FLs8th/wrIgsE5ElQF/g2OW2aSIyxtu2JZAuIouB6cCzqmoFw1Q4zWolcEWHerzz00Z2HjjidpyQtnH3Ya59/WfyCot4uF8Lt+MEJQmlgc7S0tI0PT3d7RjG+NXmPTn0eX4GN3RL4cnL2rgdJyTN25jN4Hc9vztG35JGl9RqLicKHBGZ7z1ffFJ2p7cxQS6leizXdUnmw7mb2ZKd43ackPPloq3c+MYcqsZG8cXQHhW6dlvBAAARG0lEQVSqWJSVFQxjyoF7+zQlTIS7x81n7Y6DbscJCarKS9+vZfhHi+iQUoXxQ88ktUac27GCmhUMY8qB2okxjLy+I1v35nLxS7N4dcY6CmysqVOWV1DIg58uZsS3a7iyYz3eu70rVWKj3I4V9KxgGFNO9G1dm2//cA7ntqzJc1+v4qrXfrbexinYl3OUW8bOZfyCrTxwXjOev7Y90RF2c54vrGAYU47UiI9m1I2dGHl9RzbvOWy9jTLauPswV476iYWb9/HidR0Yfl5TPHcAGF9YwTCmnBERLm1f13obZbRu1yGuGDWbvTlHef+Oblxud3GXmRUMY8op6234TlV57MvlFBYp44f2oGtDuxLqVFjBMKYcK6m3MeT9+Tb50nG+WbGDWRm7eeD8ZjS0K6FOmRUMY0LAsd7GE/1b893KnTz4yWKbF9zrSH4hT3+1gqY147mpewO345RrEW4HMMb4h4gw8MxUco4W8tzXq4iLDufvV7St8Cd1x87awJbsXN6/vRuR4fY38umwgmFMiLm7V2MO5eXzyvR1xEdH8Jd+LSts0di2P5eXp2VwQeta9GxqgwmeLisYxoSgh/o259CRAt6YuYGEmEjuO7fpyRuFoGf/u4pCVR69uJXbUUKCFQxjQpCI8NilrTmUV8iIb9cQFx3B7T0buh0roNI3ZvPloiyG9W5CcrVYt+OEBCsYxoSosDDhuavaknO0gKcmryA+OpzruqS4HSsgCouUxyctp3blGIb2LnGyTnMK7AyQMSEsIjyMFwd04OxmSfx5/FImL8lyO1JAfJq+hWVbD/BwvxbERtnfxf5iBcOYEBcdEc7rN3WmS4Nq3P/RIqav2ul2JEftz83nX1NXk9agKv3b13U7TkixgmFMBVApKpwxg9JoWacyQ96fz8y1u9yO5JiXvl9Lds5RHu/fusJeHeYUKxjGVBCVYyJ557auNKwRx6C35vH+L5vcjuR3GTsP8s5PGxnQJZk29RLdjhNyrGAYU4FUi4vi0yFncHbTGjw6YRmPT1weMmNPqSpPTFpBpahwHurb3O04IckKhjEVTEJMJGMGduGOng15+6eN3Pr2PPbn5rsd67R9t3InM9fu5oHzmlE9PtrtOCHJCoYxFVB4mPDoJa147qq2/LxuD1eOms3G3YfdjnXK8gr+N17UzWfYeFFOsYJhTAV2XZcU3r+jG9mHj3L5qNn8tG6325FOycfztrBpTw6PXtLKxotykO1ZYyq47o2qM+GeHtSIj+aWsXP5cO5mtyOVydGCIl6bsY60BlU528aLcpQVDGMMDarHMX7omfRoUoOHxy/lyUkrys3J8PELMsnaf4R7z7XpVp1mBcMYA3guux07MI3bejTkzdkbeHj8UlSDe06NgsIiRs1YR7v6ida7CAC7Z94Y86uI8DD+dmkr4qLDGTktg2a1Erjz7EZuxyrVxMVZbM7O4a+XpFnvIgAc62GISLKITBeRlSKyXESGe5c/JSJLRGSRiHwjIiXeuy8iA0Vkrfcx0Kmcxpjfe+C8ZlzUpjZ//+9Kvl+5w+04JSosUl6ZnkGL2gmc26Km23EqBCcPSRUAD6pqS6A7cI+ItAL+partVLUDMBn42/ENRaQa8BjQDegKPCYiVR3MaowpJixMeP7a9rSqU5n7PlzI6u0H3Y70O18v2866XYcZ1qcJYWHWuwgExwqGqm5T1QXe5weBlUA9VT1QbLU4oKSDpBcA36pqtqruBb4FLnQqqzHm92KjIhgzMI3Y6Ahuf2ceew7luR3pV0VFyshpa2mcFMdFbeq4HafCCMhJbxFJBToCc7yvnxGRLcCNlNDDAOoBW4q9zvQuM8YEUJ3ESrxxSxq7DuZx9/sLOFoQHFdOfb9qJ6u2H+Se3k0It95FwDheMEQkHvgcuP9Y70JVH1HVZGAcMKykZiUsK/FyDREZLCLpIpK+a1fojsBpjFs6JFfhn1e3Y+7GbB6d4P6VU6rKy9PWklIt1oYvDzBHC4aIROIpFuNUdXwJq3wAXFXC8kwgudjr+kCJM7+o6mhVTVPVtKSkpNONbIwpwWUd6nFvnyZ8kp7J2FkbXM3y49rdLM7cz9BejYmwu7oDysmrpAQYC6xU1RHFlhefjb4/sKqE5lOBviJS1Xuyu693mTHGJceunHpmykqmrXLnyilVZeT3a6mbGMOVneq7kqEic7I89wBuBvp4L6FdJCL9gGdFZJmILMFTCI5dbpsmImMAVDUbeAqY53086V1mjHHJb6+cWsSaHYG/cmrOhmzSN+1lSK/GREVY7yLQxO3jkf6Ulpam6enpbscwJqRl7cvlsldmExMZxoShPQI6lPiNY35hzY5DzPxTb2IiwwP2uaFMROarapov61qJNsaUSd0qlRh9c2d2HsjjxjFz2B2gy23nb9rL7Iw9DD6rkRULl1jBMMaUWceUqowd2IWNew5z/ehf2HnwiOOf+fK0tVSNjeTG7imOf5YpmRUMY8wp6dm0Bm8N6srWfbkMeP0Xtu93rmgs27qf6at3ccdZjYiNsiHw3GIFwxhzys5oXJ13buvKjgNHuG70z2Tty3Xkc16elkHlmAhusdn0XGUFwxhzWrqkVuO9O7qRfego143+mS3ZOX7b9pbsHB77chlfL9/OoB4NSYiJ9Nu2TdlZwTDGnLZOKVV5/45u7M/JZ8DoX9i05/TmB1+RdYDhHy2k179n8MHczVybVp+7gniY9YrCLqs1xvjNsq37uWnsHGIiwvngzm40Sor3ua2qMmdDNq/OWMcPa3YRFxXODd1SuK1nQ+okVnIwdcVWlstqrWAYY/xq5bYD3DRmDmFhwod3dqNJzYQTrl9UpHyzYgev/bCORVv2UT0uilt7pHJz91QSY+0QlNOsYBhjXLV2x0Guf2MOqsoZjatTUKgUFBWRX/xrYREFRcrug3lk7T9CcrVKDD67Mdd0rm/3WQRQWQqGXZ9mjPG7prUS+Piu7jz06WKWZx0gMlyICAvzfA0PIyJMiI2KICJcqJtYiT/3q0O/NrVtMMEgZwXDGOOIxknxfDG0h9sxjB9ZOTfGGOMTKxjGGGN8YgXDGGOMT6xgGGOM8YkVDGOMMT6xgmGMMcYnVjCMMcb4xAqGMcYYn4TU0CAisgvYdNziRGD/KSyrAez2a8CTKymXk+19Wf9E65T1Pdvvvq9v+93/7W2/l5yhgaom+dRSVUP6AYw+lWVAejBkdbK9L+ufaJ2yvmf73fa77ffyvd8rwiGpSaexLNBON0NZ2/uy/onWKet7tt99X9/2u//b2373OOUMIXVIyp9EJF19HMHR+I/td3fYfndHedvvFaGHcapGux2ggrL97g7b7+4oV/vdehjGGGN8Yj0MY4wxPrGCYYwxxidWMIwxxvjECsYpEpE4EZkvIpe4naWiEJGWIvKaiHwmIne7naeiEJHLReQNEflSRPq6naeiEJFGIjJWRD5zO8sxFa5giMibIrJTRJYdt/xCEVktIhki8mcfNvV/wCfOpAw9/tjvqrpSVYcA1wLl5lJEN/lpv09Q1TuBQcB1DsYNGX7a7+tV9XZnk5ZNhbtKSkTOBg4B76pqG++ycGANcD6QCcwDrgfCgX8ct4nbgHZ4bumPAXar6uTApC+//LHfVXWniPQH/gy8rKofBCp/eeWv/e5t9zwwTlUXBCh+ueXn/f6Zql4dqOwnEuF2gEBT1R9FJPW4xV2BDFVdDyAiHwGXqeo/gN8dchKR3kAc0ArIFZEpqlrkaPByzh/73budicBEEfkKsIJxEn76eRfgWeC/Vix846+f92BT4QpGKeoBW4q9zgS6lbayqj4CICKD8PQwrFicmjLtdxHpBVwJRANTHE0W2sq034F7gfOARBFpoqqvORkuhJX157068AzQUUQe9hYWV1nB8JASlp30WJ2qvu3/KBVKmfa7qs4AZjgVpgIp635/CXjJuTgVRln3+x5giHNxyq7CnfQuRSaQXOx1fSDLpSwVie13d9h+d0e53+9WMDzmAU1FpKGIRAEDgIkuZ6oIbL+7w/a7O8r9fq9wBUNEPgR+BpqLSKaI3K6qBcAwYCqwEvhEVZe7mTPU2H53h+13d4Tqfq9wl9UaY4w5NRWuh2GMMebUWMEwxhjjEysYxhhjfGIFwxhjjE+sYBhjjPGJFQxjjDE+sYJhXCMihwLwGf19HK7en5/ZS0TOPIV2HUVkjPf5IBF52f/pyk5EUo8fpruEdZJE5OtAZTLusIJhyj3vsNElUtWJqvqsA595onHYegFlLhjAX4CRpxTIZaq6C9gmIj3czmKcYwXDBAUR+aOIzBORJSLyRLHlE7wzGy4XkcHFlh8SkSdFZA5whohsFJEnRGSBiCwVkRbe9X79S11E3haRl0TkJxFZLyJXe5eHicgo72dMFpEpx947LuMMEfm7iPwADBeRS0VkjogsFJHvRKSWd0jrIcADIrJIRM7y/vX9uff7m1fSL1URSQDaqeriEt5rICLfe/fN9yKS4l3eWER+8W7zyZJ6bOKZGfIrEVksIstE5Drv8i7e/bBYROaKSIK3JzHTuw8XlNRLEpFwEflXsX+ru4q9PQG4scR/YBMaVNUe9nDlARzyfu0LjMYzmmcYMBk42/teNe/XSsAyoLr3tQLXFtvWRuBe7/OhwBjv80F4JlsCeBv41PsZrfDMTQBwNZ7h0sOA2sBe4OoS8s4ARhV7XZX/jZZwB/C89/njwEPF1vsA6Ol9ngKsLGHbvYHPi70unnsSMND7/DZggvf5ZOB67/Mhx/bncdu9Cnij2OtEIApYD3TxLquMZ+TqWCDGu6wpkO59ngos8z4fDDzqfR4NpAMNva/rAUvd/rmyh3MPG97cBIO+3sdC7+t4PL+wfgTuE5ErvMuTvcv3AIXA58dtZ7z363w882aUZIJ65i9ZISK1vMt6Ap96l28XkeknyPpxsef1gY9FpA6eX8IbSmlzHtBK5NfRrSuLSIKqHiy2Th1gVyntzyj2/bwH/LPY8su9zz8A/l1C26XAv0XkOWCyqs4UkbbANlWdB6CqB8DTGwFeFpEOePZvsxK21xdoV6wHlojn32QDsBOoW8r3YEKAFQwTDAT4h6q+/puFngmTzgPOUNUcEZmBZ1pcgCOqWnjcdvK8Xwsp/Wc7r9hzOe6rLw4Xez4SGKGqE71ZHy+lTRie7yH3BNvN5X/f28n4PACcqq4Rkc5AP+AfIvINnkNHJW3jAWAH0N6b+UgJ6wientzUEt6LwfN9mBBl5zBMMJgK3CYi8QAiUk9EauL563Wvt1i0ALo79PmzgKu85zJq4Tlp7YtEYKv3+cBiyw8CCcVef4NnlFIAvH/BH28l0KSUz/kJz1DY4DlHMMv7/Bc8h5wo9v5viEhdIEdV38fTA+kErALqikgX7zoJ3pP4iXh6HkXAzXjmmj7eVOBuEYn0tm3m7ZmAp0dywqupTPlmBcO4TlW/wXNI5WcRWQp8hucX7tdAhIgsAZ7C8wvSCZ/jmdxmGfA6MAfY70O7x4FPRWQmsLvY8knAFcdOegP3AWnek8QrKGEWNVVdhWcK1ITj3/O2v9W7H24GhnuX3w/8QUTm4jmkVVLmtsBcEVkEPAI8rapHgeuAkSKyGPgWT+9gFDBQRH7B88v/cAnbGwOsABZ4L7V9nf/15noDX5XQxoQIG97cGEBE4lX1kHjmUZ4L9FDV7QHO8ABwUFXH+Lh+LJCrqioiA/CcAL/M0ZAnzvMjcJmq7nUrg3GWncMwxmOyiFTBc/L6qUAXC69XgWvKsH5nPCepBdiH5woqV4hIEp7zOVYsQpj1MIwxxvjEzmEYY4zxiRUMY4wxPrGCYYwxxidWMIwxxvjECoYxxhifWMEwxhjjk/8HwzKqYH5o35oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4f1ca64ac894b55bc890ecdf42962ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      17.576413  17.692612 \n",
      "    1      16.528931  15.526525                                                                                        \n",
      "    2      15.34304   14.95686                                                                                         \n",
      "    3      14.329203  14.645228                                                                                        \n",
      "    4      13.364364  14.448114                                                                                        \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[14.448114]"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lr/2, 1, cycle_len=5, use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A bit less good than -6, but still in the same range."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As of now, the best results we have had were with a bias of -6 but perhaps we should have take one even lower, since the one that minimizes the initial loss doesn't seem to be the best choice."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_reg4 = SSD_Head(k,-8)\n",
    "models = ConvnetBuilder(f_model, 0, 0, 0, custom_head=head_reg4)\n",
    "learn = ConvLearner(md, models)\n",
    "learn.opt_fn = optim.Adam\n",
    "learn.crit = ssd_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56bc3e72a8b149ac8e49c4b35f4f401e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 14%|██████████▎                                                             | 9/63 [00:06<00:40,  1.33it/s, loss=33.2]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-8:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\site-packages\\tqdm\\_tqdm.py\", line 144, in run\n",
      "    for instance in self.tqdm_cls._instances:\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\_weakrefset.py\", line 60, in __iter__\n",
      "    for itemref in self.data:\n",
      "RuntimeError: Set changed size during iteration\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 79%|█████████████████████████████████████████████████████████▏              | 50/63 [00:29<00:07,  1.69it/s, loss=128]\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEPCAYAAABP1MOPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8FPX9x/HXJ3cIEK5wGEBuuQkYDkWsolK0tiiIiIqKImrrgUfVWn9ttbXeWo9SPPACb/GogiLWE5VAuI9wyymYcAQSIIQk398fu7QpDWQDmUw2+34+HvvI7OzMzjuTPPazM9+Z79ecc4iISOSK8juAiIj4S4VARCTCqRCIiEQ4FQIRkQinQiAiEuFUCEREIpxnhcDMEsxstpktNLOlZnbPIa8/ZWb5Xm1fRERCE+Phe+8HBjrn8s0sFphpZh8752aZWTpQz8Nti4hIiDw7InABB7/xxwYfzsyigYeB273atoiIhM7TNgIzizazBUA2MMM5lwFcD/zTObfFy22LiEhoPC0Ezrli51wa0BzoY2anAsOBp8pb18zGmllm8DHWy5wiIpHMqqqvITP7Y3DyOqAgON0SWOuca3ekdRs1auRatWrlYToRkZpn7ty525xzKeUt51ljsZmlAAecc7lmlgicCTzonGtaapn88ooAQKtWrcjMzPQqqohIjWRm60NZzsurhpoBLwcbh6OAt5xzH3m4PREROQqeFQLn3CKgZznL1PZq+yIiEhrdWSwiEuFUCEREIpwKgYhIhFMhEBGJcBFfCLJ3F7AmR33fiUjk8vLy0WqtqLiEl75bx+MzVlJYXMKDw7oztFdzv2OJiFS5iCwEc9fv5O73l5C1ZTenn5BCwYESbnlrIeu372Xcme0xM78jiohUmYgqBLl7C3nwkxW8PnsDTesmMOHSXvy8S1MOFDvuem8xT/xrFRt27OWBYd2Ij4n2O66ISJWIiELgnOPdeZv567QscvcdYMwprRl3Vgdqxwd+/bgY4+ELutOqYS0e+XQlm3P38eyoE6lXK87n5CIi3qvxhWB1dh53v7+EWWt30LNlPSad143Ox9X9n+XMjOsHtqdFg1r89u1FDB3/HS9c0ZtWjZJ8SC0iUnVq9FVDE75aw9lPfEPWljzuH9qNKdeeXGYRKG1IWiqvXt2XnXsLOX/8t2Su21FFaUVE/FGjC0HDpDh+1SOVf936M0b2aUlUVGiNwL1bNeDdX/enXq04Ln4ugw8WbPY4qYiIf6psPIJjkZ6e7vzohnrnnkKumTSX2et28PtzOnH1qW2qPIOIyNEys7nOufTylqvRRwTHqn5SHJPG9GFwl6bc/3EWizbl+h1JRKTSqRCUIz4mmoeGd6dR7XjunLKYA8UlfkcSEalUKgQhqJsQy71DurBsy24mzvzB7zgiIpVKhSBEg7s2Y1DnJjw+YyXrt+/xO46ISKVRIaiAe4d0JS46irveW0w4NLKLiIRChaACmiYncMfZHfl29XbembvJ7zgiIpVChaCCLu7Tkt6t6nPftCy25e/3O46IyDFTIaigqCjj/qHd2Lu/mHs/XOZ3HBGRY6ZCcBTaNa7Dr09vyz8X/sgXK7L9jiMickw8KwRmlmBms81soZktNbN7gvMnBuctMrN3zKy2Vxm8dN1pbWnXuDZ3v7eEPfuL/I4jInLUvDwi2A8MdM71ANKAwWbWD7jZOdfDOdcd2ABc72EGz8THRPPA0G5szt3Ho5+u9DuOiMhR86wQuICDgwHHBh/OObcbwALDgCUCYXsdZnqrBlzaryUvffcDCzeq+wkRCU+ethGYWbSZLQCygRnOuYzg/BeBrUBH4KnDrDvWzDLNLDMnJ8fLmMfk9sEdaVwngTumLFL3EyISljwtBM65YudcGtAc6GNmXYPzRwPHAVnAiMOs+6xzLt05l56SkuJlzGNysPuJ5VvzuOvdxXyzKofcvYV+xxIRCVmVjFDmnMs1sy+BwcCS4LxiM3sT+C3wYlXk8MqgLk0Z2acFr8/eyNvBG82Ob1iL7s3r0T01me7Nk+mamkxSfI0fEE5EwpBn4xGYWQpwIFgEEoFPgYeALOfc6mAbwcMAzrnbjvRefo1HUFG79h1gyeZdLNyUy+JNu1i0aRebc/cBYAbtUmrTt00DBnVuSr82DYmL0dW7IuKdUMcj8PIrajPgZTOLJnAK6i1gKvCNmdUFDFgIXOdhhiqVnBhL/3aN6N+u0b/nbcvfz+JNgeKwcGMu787bzORZG6gTH8NpHRszqHMTTjshhToJsT4mF5FIphHKqljBgWK+W7ONT5f+xIxlP7F9TyGx0cbJbRsxqEsTzurUhMZ1E/yOKSI1QKhHBCoEPioucczfsJNPl/3E9KVbWb99LwAj+7TkvvO6hjzGsohIWarDqSEpR3SUkd6qAemtGvC7szuyKjufSd+vZ9Ks9dRNiOF353TyO6KIRAAVgmrCzOjQpA73DumCGTzz9VpS6sQzZkAbv6OJSA2nQlDNmBl//GUXtuXv5y9Ts0ipE8+QtFS/Y4lIDabrF6uh6CjjsQvT6Nu6Abe9vZBvVlXfO6tFJPypEFRTCbHRPHd5Om1TanPtpLks3rTL70giUkOpEFRjdRNiefnKPtSrFcfol2azfvsevyOJSA2kQlDNNambwCtX9aG4xHHZC7PJydPwmCJSuVQIwkDblNq8cEVvsnfvZ/RLs8nXQDgiUolUCMJEz5b1GX9JL7K25HHtpLkUFqnLaxGpHCoEYeT0jo15YGg3Zq7exqXPZ7D0RzUgi8ixUyEIM8PTW/Do8B6szsnn3Kdmcsc7i8jOK/A7loiEMRWCMDTsxOZ8cdtpjDmlNe/O38TpD3/J379YTcGBYr+jiUgYUiEIU8mJsfz+F5359Oaf0b9dIx6evoIzHv2Kjxb9SDh0JCgi1YcKQZhr3SiJZy9L57Wr+1I3MZbrX5vP8Anfs3Bjrt/RRCRMqBDUECe3bcRHN5zCA0O7sW77Hob8/VtufnPBv0dIExE5HI1HUAPlFRxg/JdreGHmDzhgdP9W/Pq0diQnahQ0kUiigWmEzbn7eHT6Ct5bsJl6ibHceEZ7Lul7vMZKFokQoRYCfSLUYKn1EnlsRBofXn8KnY+ryz0fLmPQ41/x8eItalAWkX9TIYgAXVOTmXxVX14c3Zu4mCiue3UeF0z4nrnrd/odTUSqARWCCGFmnH5CY6bdOIAHhnZjw469DPvHd4z/crXf0UTEZ54VAjNLMLPZZrbQzJaa2T3B+a+a2QozW2JmL5iZWjCrUEx0FBf1acmXt53GL7o145HpK8hYu93vWCLiIy+PCPYDA51zPYA0YLCZ9QNeBToC3YBEYIyHGeQwkuJjePCC7hzfMImb3ljAzj2FfkcSEZ94VghcQH7waWzw4Zxz04KvOWA20NyrDHJkteNjeGpkT3bsKeS37yxSA7JIhPK0jcDMos1sAZANzHDOZZR6LRYYBXziZQY5sq6pydx5dkc+y/qJl79b53ccEfGBp4XAOVfsnEsj8K2/j5l1LfXyeOBr59w3Za1rZmPNLNPMMnNyNHi7l0b3b8UZHRvz12nLWbJZXVuLRJoquWrIOZcLfAkMBjCzPwIpwC1HWOdZ51y6cy49JSWlKmJGLDPj4eE9qJ8Uy42vz2ePRkATiSheXjWUYmb1gtOJwJnAcjMbA/wcGOmc0zBb1USDpDieuKgn67bv4Q8fLPU7johUIS+PCJoBX5jZImAOgTaCj4AJQBPgezNbYGZ/8DCDVEC/Ng25fmB7pszbxHvzN/kdR0SqSIxXb+ycWwT0LGO+Z9uUY3fjwHbMWrOdu99bQlqL+rRulOR3JBHxmO4slv8SEx3F3y5KIyY6ihten8f+Io16JlLTqRDI/ziuXiIPX9CdJZt389AnK/yOIyIe02kaKdOgLk254uRWTJz5A3v2F9GndQN6tazP8Q1rYWZ+xxORSqRCIId159kd2b6nkKmLtvDGnI0ANEyKo2fL+px4fH16taxH9+b1SIyL9jmpiBwLFQI5rITYaJ4a2ZPiEseq7Dzmrc9l3oadzFu/k8+yfgIgJsroclxd/virLvRqWd/nxCJyNDRCmRyVHXsKmb9hJ/M27OS9eZspcfDxTQOonxTndzQRCdIIZeKpBklxnNGpCb/9eUeevSydHXsKuX2KOq4TCUcqBHLMuqYmc8fZHZmx7Ccmz1rvdxwRqSAVAqkUV/ZvxeknpPDnqVlkbdntdxwRqQAVAqkUBzuuS06M5YbX57OvUDeiiYQLFQKpNI1qx/P4hWmsycnn3o+W+R1HREKkQiCV6pT2jbjm1La8PnsD0xZv8TuOiIRAhUAq3a2DOtCjRT3unLKITTv3+h1HRMqhQiCVLjY6iqcu6kmJg3FvLKCoWMNOiFRnKgTiiZYNa3Hf+V3JXL+TJz9f7XccETkCFQLxzJC0VC44sTlPf76KWWu3+x1HRA5DhUA8dc+vunB8wyTGvbGAHXsK/Y4jImVQIRBPJcXH8NTInuzYW8jlL8xm174DfkcSkUOoEIjnuqYm88ylJ7J8624um5jB7gIVA5HqRIVAqsTpHRvzj0tOZNmW3Vz+wmzyVAxEqg0VAqkyZ3ZuwlMje7F40y6ueHEO+fuL/I4kInhYCMwswcxmm9lCM1tqZvcE519vZqvNzJlZI6+2L9XT4K5NeWpkTxZszOXKF+ewR8VAxHdeHhHsBwY653oAacBgM+sHfAucCai/4gh1drdmPHFRGpnrd3DlS3PYW6hiIOInzwqBC8gPPo0NPpxzbr5zbp1X25XwcG7343h8RBpz1u1gzMuZ6q1UxEeethGYWbSZLQCygRnOuQwvtyfhZUhaKo8M78H3a7czdlImBQdUDET84GkhcM4VO+fSgOZAHzPrGuq6ZjbWzDLNLDMnJ8e7kOKrob2a89Cw7sxcvY2xk+aqGIj4oEquGnLO5QJfAoMrsM6zzrl051x6SkqKZ9nEf8PTW/DA0G58vTJHN52J+MDLq4ZSzKxecDqRQAPxcq+2J+FtRO+WPHFRGvM27OTCCd+zdVeB35FEIoaXRwTNgC/MbBEwh0AbwUdmdqOZbSJwumiRmT3vYQYJI0PSUnlpdB825+5j6PhvWfVTnt+RRCKCOef8zlCu9PR0l5mZ6XcMqSJLfwzccFZYVMLzl6fTu1UDvyOJhCUzm+ucSy9vOd1ZLNVOl+OSefe6k2mYFMelz2fwyZKtfkcSqdFUCKRaatGgFu9cdzKdmtXl16/OZdIs3X8o4hUVAqm2GiTF8drVfTn9hMb83/tLeGT6CsLhVKZIuFEhkGqtVlwMz4w6kRHpLXj6i9Xc/s4ijYEsUsli/A4gUp6Y6CgeGNaNJskJPPmvVcTGRHHfeV0xM7+jidQIKgQSFsyMW87qQGFRCRO+WkPrhklcfWobv2OJ1AghnRoys5vMrK4FTDSzeWY2yOtwIoe6/ecncE63pvz14yymL9XVRCKVIdQ2giudc7uBQUAKMBp4wLNUIocRFWU8dmEa3ZvX46Y35rNoU67fkUTCXqiF4ODJ2HOAF51zC0vNE6lSCbHRPH9ZOg2T4rnq5Uw25+7zO5JIWAu1EMw1s08JFILpZlYH0KUb4puUOvG8OLo3BYXFXPXSHI2BLHIMQi0EVwF3Ar2dc3sJDDIz2rNUIiHo0KQO4y/txarsfK5/bb4uKxU5SqEWgpOAFc65XDO7FLgb2OVdLJHQDGifwp+HdOWrlTnc8+Ey3XAmchRCLQT/APaaWQ/gdgLjDb/iWSqRCri4b0vGntqGSbPW88K36/yOIxJ2Qi0ERS7wVWsI8IRz7gmgjnexRCrmzsEdGdylKX+ZuowZy37yO45IWAm1EOSZ2e+AUcBUM4sm0E4gUi1ERRmPj0ije2oyN74+n+Vbd/sdSSRshFoIRgD7CdxPsBVIBR72LJXIUUiMi+a5y9NJio/hhtfms69Q4x+LhCKkQhD88H8VSDazc4EC55zaCKTaaVwngcdH9GBVdj5/nrrM7zgiYSHULiYuBGYDw4ELgQwzu8DLYCJHa0D7FK75WRtey9jAx4u3+B1HpNoLtdO53xO4hyAbAgPTA58B73gVTORY3HrWCcxas507piyie4t6pNZL9DuSSLUVahtB1MEiELS9AuuKVLm4mCieHNmTEgc3va6bzUSOJNQP80/MbLqZXWFmVwBTgWnexRI5dsc3TOK+87uSuX4nT36+2u84ItVWSKeGnHO/NbNhQH8Cnc0965x7z9NkIpVgSFoqX6/cxtOfr+Lktg3p16ah35FEqp2QT+8456Y4525xzt0cShEwswQzm21mC81sqZndE5zf2swyzGyVmb1pZnHH8guIlOfeIV04vmES495YwM49hX7HEal2jlgIzCzPzHaX8cgzs/Lu2NkPDHTO9QDSgMFm1g94EHjcOdce2EmgQzsRzyTFx/DUyJ5s37Of26csUn9EIoc4YiFwztVxztUt41HHOVe3nHWdcy4/+DQ2+HDAQP5ztdHLwHnH+DuIlKtrajJ3DO7IjGU/MXnWer/jiFQrnl75Y2bRZrYAyAZmAGuAXOdcUXCRTQTuUhbx3JX9W3PaCSn8eWoWWVvUBYXIQZ4WAudcsXMuDWgO9AE6lbVYWeua2VgzyzSzzJycHC9jSoSIijIeGd6D5MRYbnh9Prv2aTAbEaiiewGcc7nAl0A/oJ6ZHbxaqTnw42HWedY5l+6cS09JSamKmBIBGtWO528j0li/fQ9Dx3/Lhu17/Y4k4jvPCoGZpZhZveB0InAmkAV8ARzsnuJy4AOvMoiUpX+7Rky6qi/b8gs5b/y3ZK7b4XckEV95eUTQDPjCzBYBc4AZzrmPgDuAW8xsNdAQmOhhBpEy9WvTkPd+fTLJibFc/FwGHyzY7HckEd9YOFxKl56e7jIzM/2OITXQzj2FXDN5LrN/2MG4M9tz0xntMTO/Y4lUCjOb65xLL2859RckEa1+UhyTr+rLsF7N+dtnq7j5zQUUHNA4BhJZQu19VKTGiouJ4pHh3WmTksTD01ewaec+nhl1Ig1rx/sdTaRK6IhABDAzfnN6O/5+cS8Wb97F+eO/Y3V2fvkritQAKgQipfyiezPeGNuPvYVFDB3/rcY+loigQiByiJ4t6/Per/uTGBfNVS9lkp1X4HckEU+pEIiUoUWDWky8vDc79hRy9Stz2VeoBmSpuVQIRA6ja2oyT1yUxqJNudz69gJKSqr/pdYiR0OFQOQIBnVpyl1nd2La4q08OmOF33FEPKHLR0XKMWZAa9Zu28Pfv1hDq4ZJDE9v4XckkUqlQiBSDjPj3iFd2LhjL3e9t5jm9WtxUlsNeSk1h04NiYQgNjqKv1/Si+MbJnHt5LmszdE9BlJzqBCIhCg5MZYXLu9NdJRx1cuZGv9YagwVApEKaNmwFs9ddiKbc/dxzeS5FBaV+B1J5JipEIhU0InHN+DhC7oz+4cd/O7dxYRDD74iR6LGYpGjMCQtlR+27eFvn62iSd14bh/c0e9IIkdNhUDkKN10Rnuy8/Yz/ss11K8Vx9WntvE7kshRUSEQOUpmxp+HdGXX3gPcNy2L5FqxXKh7DCQMqRCIHIPoKOOxET3YXXCAO6csIjkxlp93aep3LJEKUWOxyDGKj4lmwqUn0r15PW54fT7fr9nudySRClEhEKkESfExvHhFb45vUIurX8lk8aZdfkcSCZkKgUglqZ8Ux6Sr+pKcGMvlL85mje4+ljDhWSEwsxZm9oWZZZnZUjO7KTi/h5l9b2aLzexDM6vrVQaRqtY0OYHJY/piwKjnM/gxd5/fkUTK5eURQRFwq3OuE9AP+I2ZdQaeB+50znUD3gN+62EGkSrXulESL1/Zh7yCIkZNzGCHuqKQas6zQuCc2+KcmxeczgOygFTgBODr4GIzgGFeZRDxS9fUZJ6/PJ1NO/cxamKGThNJtVYlbQRm1groCWQAS4BfBV8aDujCa6mR+rZpyD8u7cWGHXsZ/LeveeiT5RryUqolzwuBmdUGpgDjnHO7gSsJnCaaC9QByjxuNrOxZpZpZpk5OTlexxTxxMCOTfj81tP4ZY/jGP/lGs587Cs+XbpV/RNJtWJe/kOaWSzwETDdOfdYGa93ACY75/oc6X3S09NdZmamRylFqkbG2u383wdLWPlTPgM7NuZPv+xCy4a1/I4l1ZRzjvkbc+nZoh5mdlTvYWZznXPp5S3n5VVDBkwEskoXATNrHPwZBdwNTPAqg0h10rdNQ6beOIDfn9OJjLXbOevxr3jis1UUHNDpIvlfGT/sYOj475i6eIvn2/Ly1FB/YBQw0MwWBB/nACPNbCWwHPgReNHDDCLVSmx0FFef2oZ/3XoaZ3ZuwuOfrWTw377mq5U6/Sn/bdKs9SQnxnJGxyaeb8uzvoacczOBwx3PPOHVdkXCQdPkBP5+cS8u6p3DHz5YyuUvzGZor1T+cG5n6tWK8zue+Cx7dwHTl2zlipNbkRgX7fn2dGexiI8GtE/hk3EDuGFgOz5Y8CNnPf4105du9TuW+OyNORspKnFc0u/4KtmeCoGIz+Jjorl10Al88Jv+pNSO55pJc7n+tXlsz9/vdzTxQVFxCa9lbGBA+0a0bpRUJdtUIRCpJrqmJvPB9f259awOTF+6lbMe/5oPF/6oS00jzGdZ2WzdXcCoKjoaABUCkWolNjqKG85oz9QbB9CifiI3vD6fayfPJXt3gd/RpIpMnrWe45ITGNixcZVtU4VApBrq0KQOU647mbvO6ciXK3I46/Gv+WDBZr9jicfW5OQzc/U2Lu7bkpjoqvt4ViEQqaZioqMYe2pbPr5pAO0a12bcmwuYv2Gn37HEQ6/O2kBstHFh76rteUeFQKSaa5NSm5dG96Zp3QTunLKYwqISvyOJB/YWFvH23I0M7tqMxnUSqnTbKgQiYaBOQix/Oa8rK37KY8JXa/yOIx74cOGPga7Lq7CR+CAVApEwcUanJpzbvRlPf76a1dl5fseRSuSc45Xv13NCkzr0blW/yrevQiASRv70qy7Uio/mzimLKSnRZaU1xYKNuSz9cTeXnnT8UXcwdyxUCETCSKPa8dz9i85krt/Jqxnr/Y4jlWTSrPUkxUVzfs9UX7avQiASZob1SmVA+0Y88PFyjYlcA+zYU8hHi7YwtFdzasd71v3bEakQiIQZM+Ov53ejxMH/vb9Edx6HubczN1JYVMKlPjQSH6RCIBKGWjSoxa2DOvCv5dl8uMj7/urFGyUljskZ6+nTugEnNK3jWw4VApEwNbp/a3o0T+aefy5l554yR3yVau6rVTls3LHPl0tGS1MhEAlT0VHGA8O6s2vfAf48dZnfceQoTP5+PY1qx/PzLk19zaFCIBLGOjWry3WnteXdeZs1ylmY2bhjL5+vyGZknxbExfj7UaxCIBLmfnN6O9qkJHHXu4vZs7/I7zgSotdmb8CAkX1a+h1FhUAk3CXERvPgsO5szt3HX6dl6UazMLCvsJi35mzkzE5NOK5eot9xVAhEaoLerRpwZf/WvJqxgQuf+Z5VP6kLiuqq4EAxYydlsmNvIVed0trvOIAKgUiN8X/nduKR4T1YnZPPOU9+w+MzVrK/qNjvWFLK/qJirpk0l5mrt/HQsO70bdPQ70iAh4XAzFqY2RdmlmVmS83spuD8NDObZWYLzCzTzPp4lUEkkpgZF5zYnM9u+Rlnd23GE/9axS+enEnmuh1+RxMCReC6yfP4amUO95/fjeHpVTvmwJF4eURQBNzqnOsE9AN+Y2adgYeAe5xzacAfgs9FpJI0qh3PkyN78uLo3uwrLOaCCd9z9/uL2V1wwO9oEauwqITrX5vP58uzue/8rlxUDRqIS/OsEDjntjjn5gWn84AsIBVwQN3gYsnAj15lEIlkp5/QmE9vPvXfbQdnPfYV05du9TtWxDlQXMKNr89nxrKfuHdIFy7p6+/NY2WxquinxMxaAV8DXQkUg+mAEShEJzvnjtiNYnp6usvMzPQ4pUjNtWBjLndOWcTyrXmcl3YcD13Qw/dr1yNBUXEJN725gKmLtvCHcztzZRU3DpvZXOdcennLef6fYGa1gSnAOOfcbuA64GbnXAvgZmDiYdYbG2xDyMzJ0Y0yIscirUU9PrzhFMad2Z73F/zIdZPnUnBADcleKi5x3PLWQqYu2sLvz+lU5UWgIjw9IjCzWOAjYLpz7rHgvF1APeecs8AIDLucc3WP9D46IhCpPJNnrefu95cwoH0jnrssnYTYaL8j1TjFJY7fvr2Qd+dv5o7BHbnutLa+5PD9iCD4IT8RyDpYBIJ+BH4WnB4IrPIqg4j8r0v7Hc9Dw7ozc/U2Rr84h72Fuhu5Mh0oLuGOKYt4d/5mbhvUwbciUBFejoLQHxgFLDazBcF5dwFXA0+YWQxQAIz1MIOIlOHC3i2IjTFufWshV7wwhxdG9/ZtUJSa5Idtexj3xnwWbtrFuDPbc/3A9n5HColnf3nn3EwCDcJlOdGr7YpIaM7v2ZyYqCjGvbmAURMzeGl0H5ITY/2OFZacc7yduYk/fbiU2Ogoxl/Si3O6NfM7Vsj0FUAkgv2yx3HERkdxw+vzuPT5DCZd1Yd6teL8jhVWcvcW8rt3F/Pxkq2c1KYhj43oQbNk//sPqghdPyYS4QZ3bcqES09kxdY8Rj6Xwfb8/X5HChvfrd7G4L99w2dZP/G7szvy6pi+YVcEQIVARIAzOjXh+cvTWZuTz8jnZpGdV+B3pGqtsKiE+6dlccnEDGrFR/Per/tzzc/aEhV1uLPh1ZsKgYgAcGqHFF68ojcbd+xj+ITvWZ2tHkzLsjo7n/PHf8szX6/l4j4tmXrDALqmJvsd65ioEIjIv53crhGTx/Rlz/4izvv7d3y+/Ce/I1ULBQeKmbZ4C2NezmTw375my64CnrssnfvO70ZiXPjfh1ElXUwcK91QJlK1NufuY+wrmSzbsps7BnfkmlPbELg1KHKUlDgy1+/kvfmb+GjRFvIKimhcJ57zeqYy5pTWNK6b4HfEcoV6Q5muGhKR/5FaL5F3rj2Z295ZyAMfL2fF1jzuH9otIu5CXpuTz3vzN/Pe/M1s2rmPWnHRDO7SlPN7pXJy20ZEh2k7wJGoEIhImRLjonl6ZE86Na3DI5+uZO3a5+F2AAAMm0lEQVS2PTw76kSahME34YrasH0vHy/ZwrTFW1i4aRdRBv3bNeLWQR0Y1LkpSTX8ZjudGhKRck1fupWb31xA7fgYnr0snbQW9fyOdMxWZ+fzyZItTFu8lWVbdgPQLTWZX/ZoxpC01BpR8EI9NaRCICIhWb51N2NeziQ7bz8PDuvG+T2b+x2pQpxzLN+ax8dLtvLx4i2sys4HoFfLepzTrRk/79KUFg1q+ZyycqkQiEil27GnkOsmzyXjhx2M6nc8485sT8Pa8X7HKtfBxu+lP+4myqB3qwb//vBvmhz+3/wPR4VARDxxoLiEv07L4uXv1lErLoYxA1ozZkCbattp3ersfEZNzCB/fxG3D+7I2V2b0igMildlUCEQEU+tzs7j0U9X8vGSrTRIiuP609txSb+WxMdUnyuLFm/axeUvzibKjJev7E2X48L7xq+KUiEQkSqxYGMuD09fzrert5NaL5GbzmzP0J6pxET7e7/q92u2c/UrmdSrFcvkq/rSqlGSr3n84PvANCISGdJa1OPVMf2YfFVfGtaO4/Z3FjH4iW/4ZMlW/Pqi+enSrVz+4myaJSfwzrUnR2QRqAgVAhGpFKe0b8QHv+nPhEt74Zzj2slzuWDC9yzZvKtKc7wzdxPXvTqPTs3q8tY1J9XoxuDKokIgIpXGzBjctRnTx53KA0O7sW7bHn719Ezufn8xuXsLPd/+89+s5ba3F9KvTQNeG9OX+kkaWyEUKgQiUulioqO4qE9LPr/tNC47qRWvZWzg9Ee+5PXZGygpqfzTRc45Hpm+gr9MzWJwl6a8cEXvGn83cGVSY7GIeC5ry27+8MES5qzbSY/mydw7pCs9Qrw7ueBAMTv3FrJ7XxF5BQfYXXCAvIIidhcUsXtfYHp1dh6fZWUzIr0F953f1feG6upCVw2JSLXinOP9BZv567TlbMvfz4j0Ftw+uCMNgqdvdu07wOrsfNZk57M6J5/V2YHHxp17OdLHVGy0UTchlov7tuSWszpEXC+pR6JCICLVUl7BAZ74bBUvfreO2vExdG5Wl9U5+eTk/WeIzLiYKNo0SqJt49q0TalNs+QE6iTEUCchlrqlftZNjCU+Jkof/oehbqhFpFqqkxDL3ed25sLeLXjw4+Vs31PIzzqk0K5xbdql1KZd49q0aFCrRnb3XF15VgjMrAXwCtAUKAGedc49YWZvAicEF6sH5Drn0rzKISLVU4cmdZh4RW+/YwjeHhEUAbc65+aZWR1grpnNcM6NOLiAmT0KVO1FxiIi8l88KwTOuS3AluB0npllAanAMgALnNS7EBjoVQYRESlflVxjZWatgJ5ARqnZA4CfnHOrqiKDiIiUzfNCYGa1gSnAOOfc7lIvjQReP8J6Y80s08wyc3JyvI4pIhKxPC0EZhZLoAi86px7t9T8GGAo8Obh1nXOPeucS3fOpaekpHgZU0QkonlWCIJtABOBLOfcY4e8fCaw3Dm3yavti4hIaLw8IugPjAIGmtmC4OOc4GsXcYTTQiIiUnW8vGpoJlDmHSHOuSu82q6IiFRMWHQxYWY5wPpDZidT9j0Ih84va7lGwLZKC1i+w2X1av1Qli9vmVD37+Hma78f3TLa79rvR6usDMc758pvZHXOheWDwJ3K5c4vazkgszpk9Wr9UJYvb5lQ96/2u/a79nv47/dw7qv1wxDnH265qnSsGSq6fijLl7dMqPv3cPO1349uGe137fejddQZwuLUUGUzs0wXQo98Urm03/2h/e6PcNrv4XxEcCye9TtAhNJ+94f2uz/CZr9H5BGBiIj8R6QeEYiISJAKgYhIhFMhEBGJcCoEhzCzJDOba2bn+p0lUphZJzObYGbvmNl1fueJFGZ2npk9Z2YfmNkgv/NECjNrY2YTzewdv7McVGMKgZm9YGbZZrbkkPmDzWyFma02sztDeKs7gLe8SVnzVMZ+d85lOeeuJTBQUVhcbue3Strv7zvnrgauAEYcaVkJqKT9vtY5d5W3SSumxlw1ZGanAvnAK865rsF50cBK4CxgEzCHwDgI0cD9h7zFlUB3AreFJwDbnHMfVU368FUZ+905l21mvwLuBJ52zr1WVfnDVWXt9+B6jxLoKn5eFcUPW5W8399xzl1QVdmPxMsxi6uUc+7r4EhopfUBVjvn1gKY2RvAEOfc/cD/nPoxs9OBJKAzsM/MpjnnSjwNHuYqY78H3+efwD/NbCqgQlCOSvp/N+AB4GMVgdBU1v97dVNjCsFhpAIbSz3fBPQ93MLOud8DmNkVBI4IVASOToX2u5mdRmCgonhgmqfJarYK7XfgBgJjgySbWTvn3AQvw9VgFf1/bwjcB/Q0s98FC4avanohKKsb7HLPhTnnXqr8KBGlQvvdOfcl8KVXYSJIRff7k8CT3sWJGBXd79uBa72LU3E1prH4MDYBLUo9bw786FOWSKL97g/td3+E/X6v6YVgDtDezFqbWRyBkdH+6XOmSKD97g/td3+E/X6vMYXAzF4HvgdOMLNNZnaVc64IuB6YDmQBbznnlvqZs6bRfveH9rs/aup+rzGXj4qIyNGpMUcEIiJydFQIREQinAqBiEiEUyEQEYlwKgQiIhFOhUBEJMKpEEilM7P8KtjGr0LsVrwyt3mamZ18FOv1NLPng9NXmNnTlZ+u4sys1aHdKZexTIqZfVJVmcQfKgRSbQW79y2Tc+6fzrkHPNjmkfrfOg2ocCEA7gKeOqpAPnPO5QBbzKy/31nEOyoE4ikz+62ZzTGzRWZ2T6n57wdHgltqZmNLzc83s3vNLAM4yczWmdk9ZjbPzBabWcfgcv/+Zm1mL5nZk2b2nZmtNbMLgvOjzGx8cBsfmdm0g68dkvFLM/urmX0F3GRmvzSzDDObb2afmVmTYNfD1wI3m9kCMxsQ/LY8Jfj7zSnrw9LM6gDdnXMLy3jteDP7V3Df/MvMWgbntzWzWcH3vLesIywLjKQ31cwWmtkSMxsRnN87uB8WmtlsM6sT/Ob/TXAfzivrqMbMos3s4VJ/q2tKvfw+cEmZf2CpGZxzeuhRqQ8gP/hzEPAsgd4Zo4CPgFODrzUI/kwElgANg88dcGGp91oH3BCc/jXwfHD6CgKD2AC8BLwd3EZnAn3DA1xAoFvrKKApsBO4oIy8XwLjSz2vz3/uuh8DPBqc/hNwW6nlXgNOCU63BLLKeO/TgSmlnpfO/SFweXD6SuD94PRHwMjg9LUH9+ch7zsMeK7U82QgDlgL9A7Oq0ugh+FaQEJwXnsgMzjdClgSnB4L3B2cjgcygdbB56nAYr//r/Tw7lHTu6EWfw0KPuYHn9cm8EH0NXCjmZ0fnN8iOH87UAxMOeR93g3+nEtg3IKyvO8C40csM7MmwXmnAG8H5281sy+OkPXNUtPNgTfNrBmBD9cfDrPOmUBns3/3QlzXzOo45/JKLdMMyDnM+ieV+n0mAQ+Vmn9ecPo14JEy1l0MPGJmDwIfOee+MbNuwBbn3BwA59xuCBw9AE+bWRqB/duhjPcbBHQvdcSUTOBv8gOQDRx3mN9BagAVAvGSAfc75575r5mBgWjOBE5yzu01sy8JDA8KUOCcKz7kffYHfxZz+P/Z/aWm7ZCfodhTavop4DHn3D+DWf90mHWiCPwO+47wvvv4z+9WnpA7/nLOrTSzE4FzgPvN7FMCp3DKeo+bgZ+AHsHMBWUsYwSOvKaX8VoCgd9Daii1EYiXpgNXmlltADNLNbPGBL5t7gwWgY5AP4+2PxMYFmwraEKgsTcUycDm4PTlpebnAXVKPf+UQK+TAAS/cR8qC2h3mO18R6DLYgicg58ZnJ5F4NQPpV7/L2Z2HLDXOTeZwBFDL2A5cJyZ9Q4uUyfY+J1M4EihBBhFYCzdQ00HrjOz2OC6HYJHEhA4gjji1UUS3lQIxDPOuU8JnNr43swWA+8Q+CD9BIgxs0XAnwl88HlhCoFBQ5YAzwAZwK4Q1vsT8LaZfQNsKzX/Q+D8g43FwI1AerBxdRlljDrlnFtOYCjIOoe+Flx/dHA/jAJuCs4fB9xiZrMJnFoqK3M3YLaZLQB+D/zFOVcIjACeMrOFwAwC3+bHA5eb2SwCH+p7yni/54FlwLzgJaXP8J+jr9OBqWWsIzWEuqGWGs3Majvn8i0wTuxsoL9zbmsVZ7gZyHPOPR/i8rWAfc45Z2YXEWg4HuJpyCPn+ZrAYOw7/cog3lIbgdR0H5lZPQKNvn+u6iIQ9A9geAWWP5FA464BuQSuKPKFmaUQaC9REajBdEQgIhLh1EYgIhLhVAhERCKcCoGISIRTIRARiXAqBCIiEU6FQEQkwv0/ZfceDo+RwYIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a8efb28821c40baa36d1c99a9651fe4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      20.074092  17.688038 \n",
      "    1      17.585098  16.027065                                                                                        \n",
      "    2      16.097799  15.334451                                                                                        \n",
      "    3      15.018225  14.841715                                                                                        \n",
      "    4      14.134768  14.573221                                                                                        \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[14.573221]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = 1e-2\n",
    "learn.fit(lr/2, 1, cycle_len=5, use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It's not as good. So the best value seems to be close to the minimum we computed, and we can stick with that approach to pick the way to initalize the bias."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model with more anchors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "anc_grids = [4,2,1]\n",
    "# anc_grids = [4]\n",
    "anc_zooms = [0.75, 1., 1.3]\n",
    "# anc_zooms = [1.]\n",
    "anc_ratios = [(1.,1.), (1.,0.5), (0.5,1.)]\n",
    "# anc_ratios = [(1.,1.)]\n",
    "anchor_scales = [(anz*i,anz*j) for anz in anc_zooms for (i,j) in anc_ratios]\n",
    "k = len(anchor_scales)\n",
    "anc_offsets = [1/(o*2) for o in anc_grids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "anc_x = np.concatenate([np.tile(np.linspace(ao, 1-ao, ag), ag)\n",
    "                        for ao,ag in zip(anc_offsets,anc_grids)])\n",
    "anc_y = np.concatenate([np.repeat(np.linspace(ao, 1-ao, ag), ag)\n",
    "                        for ao,ag in zip(anc_offsets,anc_grids)])\n",
    "anc_ctrs = np.repeat(np.stack([anc_x,anc_y], axis=1), k, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "anc_sizes  =   np.concatenate([np.array([[o/ag,p/ag] for i in range(ag*ag) for o,p in anchor_scales])\n",
    "               for ag in anc_grids])\n",
    "grid_sizes = V(np.concatenate([np.array([ 1/ag       for i in range(ag*ag) for o,p in anchor_scales])\n",
    "               for ag in anc_grids]), requires_grad=False).unsqueeze(1)\n",
    "anchors = V(np.concatenate([anc_ctrs, anc_sizes], axis=1), requires_grad=False).float()\n",
    "anchor_cnr = hw2corners(anchors[:,:2], anchors[:,2:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop=0.4\n",
    "\n",
    "class SSD_MultiHead(nn.Module):\n",
    "    def __init__(self, k, bias):\n",
    "        super().__init__()\n",
    "        self.drop = nn.Dropout(drop)\n",
    "        self.sconv1 = StdConv(512,256, dp=drop)\n",
    "        self.sconv2 = StdConv(256,256, dp=drop)\n",
    "        self.sconv3 = StdConv(256,256, dp=drop)\n",
    "        self.out1 = OutConv(k, 256, bias)\n",
    "        self.out2 = OutConv(k, 256, bias)\n",
    "        self.out3 = OutConv(k, 256, bias)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.drop(F.relu(x))\n",
    "        x = self.sconv1(x)\n",
    "        o1c,o1l = self.out1(x)\n",
    "        x = self.sconv2(x)\n",
    "        o2c,o2l = self.out2(x)\n",
    "        x = self.sconv3(x)\n",
    "        o3c,o3l = self.out3(x)\n",
    "        return [torch.cat([o1c,o2c,o3c], dim=1),\n",
    "                torch.cat([o1l,o2l,o3l], dim=1)]\n",
    "\n",
    "class FocalLoss(BCE_Loss):\n",
    "    def get_weight(self,x,t):\n",
    "        alpha,gamma = 0.25,2.\n",
    "        p = x.sigmoid()\n",
    "        pt = p*t + (1-p)*(1-t)\n",
    "        w = alpha*t + (1-alpha)*(1-t)\n",
    "        return w * (1-pt).pow(gamma)\n",
    "\n",
    "loss_f = FocalLoss(len(id2cats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_reg4 = SSD_MultiHead(k, -4.)\n",
    "models = ConvnetBuilder(f_model, 0, 0, 0, custom_head=head_reg4)\n",
    "learn = ConvLearner(md, models)\n",
    "learn.opt_fn = optim.Adam\n",
    "learn.crit = ssd_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's do the same thing with the model having multiple anchors. We just have to change the bias on three different layers (the ones giving the ouputs of grid sizes 1, 2 and 4)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_loss(min_b=-10,max_b=0,n=100):\n",
    "    test_biases = np.linspace(min_b,max_b,n)\n",
    "    losses = np.zeros(n)\n",
    "    x,y=next(iter(md.trn_dl))\n",
    "    x,y = V(x),V(y)\n",
    "    nb1 = learn.model[8].out1.oconv1.bias.size(0)\n",
    "    nb2 = learn.model[8].out2.oconv1.bias.size(0)\n",
    "    nb3 = learn.model[8].out3.oconv1.bias.size(0)\n",
    "    fp = FloatProgress(min=0,max=n)\n",
    "    display(fp)\n",
    "    for i in range(n):\n",
    "        learn.model[8].out1.oconv1.bias = torch.nn.Parameter(torch.Tensor([test_biases[i]]*nb1).cuda())\n",
    "        learn.model[8].out2.oconv1.bias = torch.nn.Parameter(torch.Tensor([test_biases[i]]*nb2).cuda())     \n",
    "        learn.model[8].out3.oconv1.bias = torch.nn.Parameter(torch.Tensor([test_biases[i]]*nb3).cuda())\n",
    "        z = learn.model(x)\n",
    "        losses[i] = ssd_loss(z,y)\n",
    "        fp.value += 1\n",
    "    return test_biases,losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a76be0cb7a54618b3c60825ede700f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>FloatProgress</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "FloatProgress(value=0.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "b,l = test_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-3.0303030303030303"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGrVJREFUeJzt3X10XPV95/H3d2b04CdJxpafbWxjAyY8xxDTbJcFpw2wSUx6QpfdnoTT0PW2yaZJs7sNSfZst2f/2Dx0C2l3l1NOnJRQTkmahOKT0iSUhJRsa8AG82CEsRFYkiXLY1uyZEkzmofv/jF3ZAGyNbJn5s5cfV4nHM99mJnvDfjzu/c3v/u75u6IiEh0xcIuQEREKktBLyIScQp6EZGIU9CLiEScgl5EJOIU9CIiEaegFxGJOAW9iEjEKehFRCIuEXYBAIsXL/a1a9eGXYaISF3Zs2fPMXdvn26/mgj6tWvXsnv37rDLEBGpK2Z2qJT91HUjIhJxCnoRkYhT0IuIRJyCXkQk4hT0IiIRp6AXEYk4Bb2ISMQp6EVEQpDO5vjqj1/jxe7Bin+Xgl5EJARHh9Lc/9Qb7D8yXPHvUtCLiITg6HAKgCUtTRX/LgW9iEgIjpxMA7C0pbni36WgFxEJQf9Q4Yx+mYJeRCSa+odSNMZjtM1tqPh3KehFRELQP5RiSUsTZlbx71LQi4iEoH8oXZVuG1DQi4iEon8oVZUfYkFBLyISimLXTTUo6EVEquxUOsvIeE5dNyIiUXXkZGFopbpuREQi6uiQgl5EJNL6h4tBrz56EZFIqub0B6CgFxGpuv6hFAuaEsxrSlTl+xT0IiJVVs2hlaCgFxGpumreLAUKehGRqqvm9AdQYtCb2R+Y2T4ze8XM/trMms1snZk9Y2YHzOy7ZtYY7NsULB8Mtq+t5AGIiNSTfN45OpxiSS0FvZmtBH4f2OzulwNx4E7gq8C97r4RGADuDt5yNzDg7huAe4P9REQEGBgdJ5Pzqg2thNK7bhLAHDNLAHOBPuBm4PvB9geB24PX24Jlgu1brRrzcIqI1IH+ocLQyprqunH3w8CfAF0UAv4ksAcYdPdssFsPsDJ4vRLoDt6bDfZfVN6yRUTqU/HJUrXWdbOQwln6OmAFMA+4dYpdvfiWs2yb/LnbzWy3me1OJpOlVywiUsf6h6p7VyyU1nXzAeBNd0+6ewb4IfArQFvQlQOwCugNXvcAqwGC7a3AiXd+qLs/4O6b3X1ze3v7eR6GiEh9KHbdLFlQQ2f0FLpstpjZ3KCvfSvwKvBz4GPBPncBjwWvdwbLBNt/5u7vOqMXEZmNjgylWDSvkcZE9Ua3l9JH/wyFH1WfB14O3vMA8AXg82Z2kEIf/I7gLTuARcH6zwP3VKBuEZG6dHSoukMroTCaZlru/kfAH71jdSdw/RT7poA7zr80EZHo6R9OsayK/fOgO2NFRKrqyMl0Vac/AAW9iEjVZHJ5jo+kq951o6AXEamS5HAa9+reLAUKehGRqukLnhW7rFV99CIikdQ7OAbAyra5Vf1eBb2ISJUcDoJ+RZu6bkREIql3cIyW5gQLmhuq+r0KehGRKukdHGNF25yqf6+CXkSkSnoGxlipoBcRia7ewTFWLlTQi4hE0nAqw1Aqq64bEZGo6h0sjKFX142ISET1TgytVNCLiERSz8TNUgp6EZFI6h0coyFuLFlQ3ekPQEEvIlIVhwfGWNbaTCw21WO1K0tBLyJSBb2D4YyhBwW9iEhVhHVXLCjoRUQqLpPLc2QoxSoFvYhINPUPpch7OEMrQUEvIlJxhwfCG0MPCnoRkYrrPRmMoQ9hnhtQ0IuIVFxx+oMVrQp6EZFI6hkYY9G8RuY0xkP5fgW9iEiFhTm0EhT0IiIVd3hwrOrPiZ1MQS8iUkHuHtwVOze0GhT0IiIVdHIsw+h4Tmf0IiJR1ROMoV8V0tBKUNCLiFRUMejVdSMiElHdJ0YBWHOBgl5EJJK6B0ZpaU7QOrchtBoU9CIiFdR1YpQ1i8I7mwcFvYhIRXWdGGX1QgW9iEgk5fNOz8BYqP3zoKAXEamYo8NpxrN5VtVD0JtZm5l938xeM7MOM7vBzC4wsyfM7EDw58JgXzOzPzOzg2b2kpldW9lDEBGpTV01MOIGSj+j/wbwY3e/FLgK6ADuAZ50943Ak8EywK3AxuCf7cD9Za1YRKRO1MLQSigh6M2sBfiXwA4Adx9390FgG/BgsNuDwO3B623Ad7xgF9BmZsvLXrmISI3rOjGKGaFOfwClndGvB5LAt83sBTP7ppnNA5a6ex9A8OeSYP+VQPek9/cE60REZpXuE6Msb2mmKRHOPPRFpQR9ArgWuN/drwFGON1NMxWbYp2/ayez7Wa228x2J5PJkooVEakn3QOjrA652wZKC/oeoMfdnwmWv08h+PuLXTLBn0cn7b960vtXAb3v/FB3f8DdN7v75vb29nOtX0SkZnWdqJOgd/cjQLeZXRKs2gq8CuwE7grW3QU8FrzeCXwiGH2zBThZ7OIREZktUpkc/UPp0H+IhUK3TCk+AzxsZo1AJ/DbFBqJ75nZ3UAXcEew7+PAbcBBYDTYV0RkVinOWlk3Qe/ue4HNU2zaOsW+Dnz6POsSEalrxaGVqy8Ibx76It0ZKyJSAV0TQR/+Gb2CXkSkArpPjNLcEKN9flPYpSjoRUQqoThrpdlUI86rS0EvIlIBXSdGa+KHWFDQi4iUnXtheuJa6J8HBb2ISNkNjGY4lc4q6EVEoqpWpicuUtCLiJTZoeMjgIJeRCSy3jw2ghlcGPJDwYsU9CIiZdaZHGFl2xyaG8KdnrhIQS8iUmadx06xvn1+2GVMUNCLiJSRu/NmcoT1i+eFXcoEBb2ISBn1D6UZGc9xUbuCXkQkkjqTpwDUdSMiElVvHCsMrVyvM3oRkWjqTJ5ibmOcZS3NYZcyQUEvIlJGnckR1i2eVxOzVhYp6EVEyujNYyM11T8PCnoRkbJJZ3P0DIzW1NBKUNCLiJTNoeOj5L22fogFBb2ISNlMDK1crK4bEZFIeiNZGFq5Tmf0IiLR1JkcYWlLE/ObEmGX8jYKehGRMuk8dqrmum1AQS8iUhbuTmdypOZ+iAUFvYhIWQyMZjg5lqm5MfSgoBcRKYvTk5npjF5EJJLeCIL+IvXRi4hEU0ffMHMb46xaOCfsUt5FQS8iUgYdfUNcsmwBsVjtTGZWpKAXETlP7s5rR4bZtLwl7FKmpKAXETlPfSdTnBzLKOhFRKKqo28IgMuWLwi5kqkp6EVEzlMx6C9ZpjN6EZFI6jgyzJoL5tbcHDdFCnoRkfPU0TfEphrttoEZBL2Zxc3sBTP7UbC8zsyeMbMDZvZdM2sM1jcFyweD7WsrU7qISPjGxnO8dWyES2u02wZmdkb/WaBj0vJXgXvdfSMwANwdrL8bGHD3DcC9wX4iIpG0v3+YvFOzI26gxKA3s1XAvwa+GSwbcDPw/WCXB4Hbg9fbgmWC7Vutlh6HLiJSRq9NjLip86AH7gP+EMgHy4uAQXfPBss9wMrg9UqgGyDYfjLY/23MbLuZ7Taz3clk8hzLFxEJV0ffEPObEjU59UHRtEFvZh8Cjrr7nsmrp9jVS9h2eoX7A+6+2d03t7e3l1SsiEit6egbrtmpD4pKOaN/P/ARM3sLeIRCl819QJuZFccSrQJ6g9c9wGqAYHsrcKKMNYuI1AR3p+NIbY+4gRKC3t2/6O6r3H0tcCfwM3f/LeDnwMeC3e4CHgte7wyWCbb/zN3fdUYvIlLvDg+OMZzK1vQPsXB+4+i/AHzezA5S6IPfEazfASwK1n8euOf8ShQRqU0dfcMANT20EmBGt3G5+1PAU8HrTuD6KfZJAXeUoTYRkZr2au8QZnDpsjrvuhERkant7R5g45L5zKvRqQ+KFPQiIufA3dnbPcjVq9vCLmVaCnoRkXPQdWKUgdEMV69eGHYp01LQi4icg73dgwA6oxcRiaoXugaZ0xDn4qXzwy5lWgp6EZFzsLd7kCtWtZKI136M1n6FIiI1Jp3N8WrvENfUQbcNKOhFRGaso2+Y8Vy+LvrnQUEvIjJje7sGALh6jYJeRCSS9nYPsrSlieWttTs18WQKehGRGaqXG6WKFPQiIjMwMDLOW8dH6+JGqSIFvYjIDOztqZ8bpYoU9CIiM/BC1yAxgytXtYZdSskU9CIiM/BM53EuW9FS8zNWTqagFxEpUSqT44XuQW5YvyjsUmZEQS8iUqLnuwYYz+bZoqAXEYmmXW8cJ2Zw3boLwi5lRhT0IiIl2tV5gitWttLS3BB2KTOioBcRKcHYeI4XugfqrtsGFPQiIiXZc2iATM7ZcpGCXkQkknZ1HiceM65bW1/986CgFxEpyT93HueKla3Mr6Px80UKehGRaYyks7zYPcgNddhtAwp6EZFp7Tk0QDbvdXejVJGCXkRkGv/ceZxEzHjvhfUzY+VkCnoRkWk8fSDJNWva6mp+m8kU9CIiZ3F0OMUrh4f4V5csCbuUc6agFxE5i6dfPwbAjRe3h1zJuVPQi4icxVOvJ1k8v4nLlreEXco5U9CLiJxBLu88fSDJjRe3E4tZ2OWcMwW9iMgZvNgzyOBohhsvqd9uG1DQi4ic0S/2J4kZ/OqGxWGXcl4U9CIiZ/CL15NctbqNhfMawy7lvCjoRUSmcGJknBd7But6tE3RtEFvZqvN7Odm1mFm+8zss8H6C8zsCTM7EPy5MFhvZvZnZnbQzF4ys2srfRAiIuX29IEk7tT1+PmiUs7os8B/cvdNwBbg02Z2GXAP8KS7bwSeDJYBbgU2Bv9sB+4ve9UiIhX21P4kC+c2cMXK1rBLOW/TBr2797n788HrYaADWAlsAx4MdnsQuD14vQ34jhfsAtrMbHnZKxcRqZCx8Rw/3XeEX7tsKfE6HlZZNKM+ejNbC1wDPAMsdfc+KDQGQPH6ZiXQPeltPcE6EZG68ERHPyPjOT56zaqwSymLkoPezOYDPwA+5+5DZ9t1inU+xedtN7PdZrY7mUyWWoaISMU9+nwPK1qbed+6+nua1FRKCnoza6AQ8g+7+w+D1f3FLpngz6PB+h5g9aS3rwJ63/mZ7v6Au292983t7fX/q7aIRMOxU2n+8cAxtl2zsq7vhp2slFE3BuwAOtz9Tydt2gncFby+C3hs0vpPBKNvtgAni108IiK17kcv9pLLOx+9Jjo9zqVMrvx+4OPAy2a2N1j3JeArwPfM7G6gC7gj2PY4cBtwEBgFfrusFYuIVNCje3u5bHkLFy9dEHYpZTNt0Lv7L5m63x1g6xT7O/Dp86xLRKTqOpOneLF7kC/ftinsUspKd8aKiAT+9oXDxAw+cvWKsEspKwW9iAiFKYl/8Pxh3r9hMUtbmsMup6wU9CIiwJMd/RweHOO33rcm7FLKTkEvIgI8tOsQy1qa+cCmpWGXUnYKehGZ9TqTp3j6wDH+3fvWkIhHLxajd0QiIjP0V7u6SMSMO69fPf3OdUhBLyKz2uh4lr/Z080tly9jyYJo/QhbpKAXkVlt595ehlNZPnHD2rBLqRgFvYjMWrm88+3/9xaXLF3AdWsXhl1OxSjoRWTWevSFw+zvH+ZTN11EYVqvaFLQi8islMrk+F8/3c+Vq1r58JXRuhP2nRT0IjIr7fjlm/SdTPGl2zZFZjriM1HQi8isc/xUmvufeoMPbFrClvWLwi6n4hT0IjLr/PnPDjKWyXHPrZeGXUpVKOhFZFbp6BvioV2HuPO61WxYEp05589GQS8is0Y+73zxhy/TNqeB//LBS8Iup2oU9CIyazz8bBd7uwf5rx/aRNvcxrDLqRoFvYjMCkeHUnztx6/x/g2LuP3q6DwPthQKehGZFf74R6+Szub5H9suj/TNUVNR0ItI5P3VrkP83Ut9fOamDaxvnx92OVWnoBeRSHv2zRP89537uOmSdj5104awywmFgl5EIqt3cIxPPbyHNRfM5b47ryEe8TtgzyQRdgEiIpVwKp3lPzy0h1QmzyPb30vrnIawSwqNgl5EImd0PMsnv/0cr/YN8cDH3ztrbow6E3XdiEikjI3n+ORfPsfuQye4799czdYIPux7phT0IhIZp9JZfuc7z/HMmyf409+8mg9fFe3ph0ulrhsRiYTO5Cm2P7SHN4+N8PWPXcXt18yum6LORkEvInXvyY5+PvfIXhJx46FPXs+vbFgcdkk1RUEvInVrdDzL1368n7/8p7d4z4oW/uLj72XVwrlhl1VzFPQiUpf+6Y1jfOEHL9F9Yoy7briQe27dxJzGeNhl1SQFvYjUlY6+Ie77h9f5yb5+Llw0l+9u38L7ZsFTos5HXQd9/1CKvpMpGuMxGhMxmhIxGuIxGuJGIh4jHjNiBjEzmhKxWTeRkUhUuDvPdw2y45edPP7yERY0Jfjs1o387o0X6Sy+BHUd9I++cJiv/P1rJe/fmIjRnCg0ColYjETciMeMuBlm0JiI09wQozkRJxE3zE43FDEzEjGjIRFjTkOM5oY4TYkYiXjQuMSMWKywTzxmNDXEaQoaoES8uD42sT0RNxqL740X9il+x9u2JQoNV0MsFvkHGIu807FTaX70Yi9//Ww3+/uHmd+U4DM3b+B3/sV6WufO3jtdZ6qug/62y5ezccl8Mrk86Wye8WyebN7J5Aqv3SHvTjbvpLN50tkc6Uye8VyebC5PNufk3Ml74ckzxX1SmRzpbLDenbw7uTzk8nkyOSeVyTGWKXxWNlhXDfGgIWkIrlYa4jbRYJ1eF6MxbjQGDVpjPEZT0HAVrnDsXfsVG6vG4HMmGpiYBdsK3zPxnfHT25obYsxpiNPcECcWXEEZNnGF1dwQn7Xzi8jMZXJ59h8ZZlfncX6y7wi7Dw3gDleuauV//sYVfPiqFcxvquvYCkVd/z+2ZtFc1iwK/xd2DxqTXL7QKGSyTjqXYzxofHL5wvZszsnmC8uZ4PV4Nk8mlyeXZ2JbNueM5wrri68LjVhhufjewmfmg8/1iUan2NClM3mGU1nSmcJn5YNGLZvLMz5pv0o3VmaQCBqZYtdasUFpaohNdL3FY8UrF0gEVzBxY6KBKl7pNCXiNE7qpmuIFxqVpobTjVoiZm+7akvEio1PfKKLr9hAFhvPRNCgNcRPN4bF96rbr/zGs3neSJ7i5cMneeXwSV4+fJJ9vUOMZ/MAXLpsAb9/80ZuuXwZm5a3hFxtfavroK8VZhYETrCiEaC+LivdTzcgxYYqmys0Opl8fqKRKv6ZCbalMjlS2Rxj4zny7sFVFIxnc6SzeVLBVU8uaAgnX3EVG7BiY1dsDHN5ZyyXI5t38pMa0GzeCw1YttCITm4Q8xW+qIoZE1c1ibhNNBhNQQNVbBiaJ7r14m9rMBqCq6eJK6ZJV0nxiUbw9D5NidhEw1Ts7osHjVVzIj7RQBYbv8Skq7zi58WCrsdqNlLj2Txj4zlGM1kGRzMcPzXO8ZE0yeHCP0eH0xweHKPnxCh9Qyk8+Pc2rzHOe1a08oktF3Ll6jauXdOmYZJlVJGgN7NbgG8AceCb7v6VSnyPlI+Z0ZgwGonVZUNV7L5LZ3KFK6bgaqfYwGSCRiGdyZHKFrruMpMar9MNTX7S1ZdPdNfl8oVuvtxEY3O6wcm7T3xGKpPnVDrLsVPj5IqfHTSe47k8mWy+sBx8ZjVMNESx0116DfHYREMQM4PC/87YKLj7265E8w7uxROE/MT/v2c7psZEjCULmlje2syWixaxeuFc1rfP4/KVraxbNE+/QVVQ2YPezOLA/wF+DegBnjOzne7+arm/S6SoeKZcT/23xfDMuZPPEzQIp6900kH32+QrovGgMUllcm/r2ptooIL9vNhNF1wVFa/KJl9BOUxcLTmAg+MYUwdusRus2L1mVtizIX7696DmhhhzGhPMbYzTNqeBC+Y1smh+I4vnN9E6p0FdYCGpxN+K64GD7t4JYGaPANsABb3IJBb87lD8SzgHDROUyqjE7JUrge5Jyz3BOhERCUElgn6qa7N3ddyZ2XYz221mu5PJZAXKEBERqEzQ9wCrJy2vAnrfuZO7P+Dum919c3t7ewXKEBERqEzQPwdsNLN1ZtYI3AnsrMD3iIhICcr+Y6y7Z83sPwI/oTC88lvuvq/c3yMiIqWpyFg0d38ceLwSny0iIjOjZ8aKiEScgl5EJOLMvTq3YZ+1CLMkcOgc374YOFbGcuqBjnl20DHPDudzzBe6+7TDFmsi6M+Hme12981h11FNOubZQcc8O1TjmNV1IyIScQp6EZGIi0LQPxB2ASHQMc8OOubZoeLHXPd99CIicnZROKMXEZGzqNugN7M7zGyfmeXNbPM7tn3RzA6a2X4z+2BYNVaSmV1tZrvMbG8wC+j1YddUDWb2meDf6z4z+1rY9VSLmf1nM3MzWxx2LZVmZl83s9fM7CUze9TM2sKuqRLM7Jbgv+WDZnZPJb+rboMeeAX4DeAfJ680s8soTKT2HuAW4P8GT72Kmq8Bf+zuVwP/LViONDO7icJDbK509/cAfxJySVVhZqspPLGtK+xaquQJ4HJ3vxJ4HfhiyPWU3aQn8d0KXAb82yC7KqJug97dO9x9/xSbtgGPuHva3d8EDlJ46lXUONASvG5liqmgI+j3gK+4exrA3Y+GXE+13Av8IVM81yGK3P2n7p4NFndRmOo8aiaexOfu40DxSXwVUbdBfxaz5QlXnwO+bmbdFM5sI3fWM4WLgV81s2fM7Bdmdl3YBVWamX0EOOzuL4ZdS0g+Cfx92EVUQFVzqqafpGxm/wAsm2LTl939sTO9bYp1dXkmdLbjB7YCf+DuPzCz3wR2AB+oZn2VMM0xJ4CFwBbgOuB7Zrbe63zo2DTH/CXg16tbUeWV8nfbzL4MZIGHq1lblVQ1p2o66N39XIKrpCdc1YOzHb+ZfQf4bLD4N8A3q1JUhU1zzL8H/DAI9mfNLE9hnpC6fhblmY7ZzK4A1gEvmhkU/lt+3syud/cjVSyx7Kb7u21mdwEfArbWe0N+BlXNqSh23ewE7jSzJjNbB2wEng25pkroBW4MXt8MHAixlmr5WwrHipldDDQS4Qmw3P1ld1/i7mvdfS2FcLi23kN+OmZ2C/AF4CPuPhp2PRVS1Sfx1fQZ/dmY2UeBPwfagb8zs73u/kF332dm3wNepXDZ92l3z4VZa4X8e+AbZpYAUsD2kOuphm8B3zKzV4Bx4K6Inu3Ndv8baAKeCK5kdrn774ZbUnlV+0l8ujNWRCTioth1IyIikyjoRUQiTkEvIhJxCnoRkYhT0IuIRJyCXkQk4hT0IiIRp6AXEYm4/w+xtX1n8rL5jgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(b,l)\n",
    "b[l.argmin()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks like here -3 is the minimum."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_reg4 = SSD_MultiHead(k, -3.)\n",
    "models = ConvnetBuilder(f_model, 0, 0, 0, custom_head=head_reg4)\n",
    "learn = ConvLearner(md, models)\n",
    "learn.opt_fn = optim.Adam\n",
    "learn.crit = ssd_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d596e17cf74b4c498311f5bfed8752de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 75%|████████████████████████████████████████████████████▉                  | 47/63 [00:29<00:10,  1.60it/s, loss=51.8]"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEOCAYAAABxdpuaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd8VfX9x/HXJwlhJSSMsMOI7L2HAg4Uce9VFypa25/Vn9Y6aqut/qpWa1srVqtinVAnqAiIohZRQRJmSNgQRgIJkAGEAEm+vz/uRSMmkBty70lu3s/HI4/ce3LGJzmQd875fs/3a845REREAhHhdQEiIlL7KDxERCRgCg8REQmYwkNERAKm8BARkYApPEREJGAKDxERCZjCQ0REAqbwEBGRgCk8REQkYFFeF1CdWrRo4Tp16uR1GSIitUZKSspO51xCoNuFVXh06tSJ5ORkr8sQEak1zCyjKtvptpWIiARM4SEiIgFTeIiISMAUHiIiEjCFh4iIBEzhISIiAVN4iIjUYqnb8vl2/S5KS0M7pbjCQ0SkFnv56438aupizEJ7XIWHiEgttjgjl0EdmmIhTg+Fh4hILZWz5wCbdhUypFPTkB9b4SEiUkst3pwLwOCOCg8REamklIxcoiMj6NMuLuTHVniIiNRSKRm59G0fR/2oyJAfW+EhIlILFR0qYcXWfIZ4cMsKFB4iIrXSysx8DpaUMkjhISIilZW8ybvGclB4iIjUSikZuXRq3ogWMfU9Ob7CQ0SklnHOkZKRy+COzTyrIWjhYWYvm1m2maWWWfaImS03s6VmNsfM2paz3QAz+9bMVvrXvSJYNYqI1EYZuwrZte+gZ7esILhXHq8A449Y9qRzrp9zbgAwA3iwnO0Kgeucc7392//dzOKDWKeISK2SnOFr7/DiyfLDooK1Y+fcPDPrdMSygjJvGwM/GQbSObemzOtMM8sGEoC84FQqIlK7pGTk0qRBFF0SYjyrIWjhUREz+xNwHZAPnHqMdYcB0cD6EJQmIlIrpGTsZlDHpkREhHgo3TJC3mDunHvAOZcIvAncVtF6ZtYGeB24wTlXepT1bjGzZDNLzsnJqf6CRURqkPz9h1izYy+DO3h3ywq87W01BbikvC+YWRPgY+B3zrkFR9uJc+4F59wQ59yQhISEIJQpIlJzfD8YooftHRDi8DCzrmXeng+sKmedaGAa8Jpz7p1Q1SYiUhukbMolMsIYkOhtP6KgtXmY2VTgFKCFmW0FHgLONrPuQCmQAdzqX3cIcKtzbiJwOTAGaG5mE/y7m+CcWxqsWkVEaouUjFx6tWlCo+iQN1n/SDB7W11VzuLJFaybDEz0v34DeCNYdYmI1FaHSkpZuiWPK4Ymel2KnjAXEaktVmXtYf+hEk8fDjxM4SEiUkskZ+wGvH048DCFh4hILZGSkUvbuAa0iWvodSkKDxGR2iIlI5fBnbwbDLEshYeISC2QmbefrPwiBneoGUP9KTxERGqBHwZD1JWHiIhU0uKMXBpFR9KjdazXpQAKDxGRWiE5YzcDEuOJiqwZv7ZrRhUiIlKhfQeKSc/aUyOe7zhM4SEiUsMt25JHSalTeIiISOWlZORiBgM9Hoa9LIWHiEgNl5yRS7eWscQ1rOd1Kd9TeIiI1GClpY7Fm3MZVINuWYHCQ0SkRlubvZc9RcUMUXiIiEhlHR4MsSY1loPCQ0SkRkvJyKVFTDQdmzfyupQfUXiIiNRgKRm5DOrQFDPzupQfUXiIiNRQOXsOkLGrsEbM33EkhYeISA2V4h8Msaa1d4DCQ0Skxlq8OZfoyAj6tIvzupSfUHiIiNRQyZt207d9HPWjIr0u5ScUHiIiNVDRoRJStxXUuOc7DlN4iIjUQKnb8jlYUloj2ztA4SEiUiO9tWgLURGm8BARkcpZuGEX76RsZeLoJJrH1Pe6nHIpPEREapCDxaU8MD2V9k0bcsfYrl6XU6EorwsQEZEfvPjVBtZl7+XfE4bSMLrm9bI6TFceIiI1xOZdhfxj7lrO6tOaU3u09Lqco1J4iIjUAM45fv9BKvUiI3jovN5el3NMCg8RkRrg4xVZ/HdNDr8e143WcQ28LueYFB4iIh4rKDrEHz9Ko0+7Jlw3spPX5VSKGsxFRDz21Cer2bX3AJOvH0JkRM0aer0iuvIQEfHQsi15vLYgg+tGdqJf+3ivy6k0hYeIiEeKS0r57bQVJMTU565x3bwuJyAKDxERj7z2bQYrMwt46LzeNGlQz+tyAhLU8DCzl80s28xSyyx7xMyWm9lSM5tjZm0r2PZ6M1vr/7g+mHWKiIRaVv5+npqzmlO6J3B239ZelxOwYF95vAKMP2LZk865fs65AcAM4MEjNzKzZsBDwHBgGPCQmdXM0cFERKrgjx+mUVzqePj8PjVufvLKCGp4OOfmAbuPWFZQ5m1jwJWz6ZnAp8653c65XOBTfhpCIiK10tz0HcxeuZ3bx3alQ/NGXpdTJZ501TWzPwHXAfnAqeWs0g7YUub9Vv8yEZFabf/BEh78YCVdW8Zw8+gkr8upMk8azJ1zDzjnEoE3gdvKWaW8a7jyrlAws1vMLNnMknNycqqzTBGRavfN+p1sy9vPb8/pSXRU7e2z5HXlU4BLylm+FUgs8749kFneDpxzLzjnhjjnhiQkJAShRBGR6pOWWYAZDO3UzOtSjkvIw8PMyg5Qfz6wqpzVPgHGmVlTf0P5OP8yEZFabWVmAZ2aNyamfu0e4COo1ZvZVOAUoIWZbcXXg+psM+sOlAIZwK3+dYcAtzrnJjrndpvZI8Ai/64eds7t/skBRERqmbSsAvq2i/O6jOMW1PBwzl1VzuLJFaybDEws8/5l4OUglSYiEnIFRYfYvLuQK4YmHnvlGs7rNg8RkTojPdP3pEKvtk08ruT4KTxEREJkpT88eis8RESkstKyCmgRU5+WsTV/sqdjUXiIiIRIWmZBWFx1gMJDRCQkDhaXsjZ7T1i0d4DCQ0QkJNZm7+FQidOVh4iIVN7hxvJebRQeIiJSSWmZBTSKjqRT88Zel1ItFB4iIiGQlllAzzZNiIiofXN3lEfhISISZKWljrSs8OlpBQoPEZGg25JbyN4DxWHT3gEKDxGRoEsLo2FJDlN4iIgE2crMAiIjjG6tYr0updooPEREgiwtq4AuCTE0qBfpdSnVRuEhIhJkKzPzw6qxHBQeIiJBtXPvAXYUHAir9g5QeIiIBFV6Vvg1loPCQ0QkqMJtWJLDFB4iIkGUlllAu/iGxDeK9rqUaqXwEBEJopWZ+WF3ywoUHiIiQVN4sJgNO/eFXU8rUHiIiATNqu17cC782jtA4SEiEjThOCzJYQoPEZEgWZlZQFzDerSLb+h1KdVO4SEiEiRpWQX0atMEs/CYw6MshYeISBAUl5SyKszm8ChL4SEiEgQbd+7jQHFpWLZ3gMJDRCQoDj9Z3rttnMeVBEelwsPM7jCzJuYz2cwWm9m4YBcnIlJbpWUVEB0VQVJCY69LCYrKXnnc6JwrAMYBCcANwONBq0pEpJZLyyygR+tY6kWG5w2eyn5Xh7sKnA382zm3rMwyEREpwznnG5YkDB8OPKyy4ZFiZnPwhccnZhYLlAavLBGR2mt7QRG5hYfCtqcVQFQl17sJGABscM4VmlkzfLeuRI5pT9Eh9h0ooXVcA69LEQmJldvC98nywyp75TESWO2cyzOza4DfAfnBK0vCgXOOGcszOfUvXzL+6XnsPVDsdUkiIZGWVYAZ9Git8HgOKDSz/sA9QAbwWtCqklpve34RN7+Wwm1TlhDfKJq8wkO8vWiL12WJhMTKzHw6N29M4/qVvblT+1Q2PIqdcw64AHjaOfc0EHu0DczsZTPLNrPUMsueNLNVZrbczKaZWXwF295pZivNLNXMppqZ7nfUEqWljjcXZnDGX//L/HU5PHB2T2bfMZphnZoxef5GikvUVCbhLy2rgJ5hfMsKKh8ee8zsfuBa4GMziwTqHWObV4DxRyz7FOjjnOsHrAHuP3IjM2sH3A4Mcc71ASKBKytZp3hoQ85ernpxAQ9MS6Vv+zg++d8x3DwmiajICG4ek8S2vP3MSt3udZkiQZW//xBbdu8P68ZyqHx4XAEcwPe8x3agHfDk0TZwzs0Ddh+xbI5z7vCN7wVA+wo2jwIamlkU0AjIrGSd4oFDJaX888t1jH/6K9KyCvjzJX15c+JwOjb/4eGosT1aktSiMS/M24DvIlYkPKVnheec5UeqVHj4A+NNIM7MzgWKnHPH2+ZxIzCrnGNtA/4CbAaygHzn3JzjPJYESeq2fC589muemL2a07q3ZO5dJ3PF0A4/GUU0IsKYODqJFdvyWbhxdwV7E6n9wn1YksMq1ZpjZpfju9L4Et/Dgc+Y2W+cc+9W5aBm9gBQjC+QjvxaU3xtK52BPOAdM7vGOfdGBfu6BbgFoEOHDlUpRyqhtNSxJbeQtTv2siZ7D+v8n9Oz9tCscTTPXzOI8X3aHHUfFw9qx1NzVvPivA2MSGoeospFQists4CE2PokxNb3upSgqmxXgAeAoc65bAAzSwA+AwIODzO7HjgXGOvKv39xOrDROZfjX/994ESg3PBwzr0AvAAwZMgQ3Q+pBs45vl63i2Vb81iXvZc1O/awPmcvRYd+aOxuE9eALi1j+MXJJ3Dz6CTiGh2rCQwa1IvkupGd+Ntna1iXvYcuLY/a50KkVlqZmR/27R1Q+fCIOBwcfruowoi8ZjYeuBc42TlXWMFqm4ERZtYI2A+MBZIDPZZU3dvJW7j3vRXADyExIqkj3VrF0KVlLF1bxdCkwbHDojzXjuzIP79cx0tfbeTxS/pVZ9kinjtQXMK67L2c1qOl16UEXWXDY7aZfQJM9b+/Aph5tA3MbCpwCtDCzLYCD+HrXVUf+NR/T3yBc+5WM2sLvOScO9s5t9DM3gUW47u1tQT/lYUEX+6+gzw+axVDOzVl8oShVQ6JijRrHM1lQ9rz9qKt3DWuGy1j1QtbwsfaHXspLnVh394BlQwP59xvzOwS4CR8bR4vOOemHWObq8pZPLmCdTPxjZt1+P1D+MJGQuyJT1ZRUFTMIxf2qfbgOOymUUm8uXAzr3+bwa/HdQ/KMUS8kJYZ/sOSHFbpW0/Oufecc3c55+48VnDIsX2zfie3T11CQdEhr0v53uLNuUz9bgs3nNgpqMMqdG7RmHG9WvH6ggwKD2rIEgkfaVkFNI6OpGOzRl6XEnRHDQ8z22NmBeV87DGzglAVGY6e/WIdHy7LZOKryRQdKvG6HIpLSvndtFRaNanP/57RLejHu2VMEnmFh3g3ZWvQjyUSKisz8+nZpgkREeE/Y8VRw8M5F+uca1LOR6xzLvyvy4Jk594DfLt+F4M7NmXRpt3cNmWx58N2vLEgg7SsAh48tzcxIRiPZ3DHZgzqEM9LX22kpFSd5KT2Ky11pGftqRO3rEBzmHtidup2Sh3834V9ePj83nyWns29762g1KNfotkFRTw1Zw2ju7bg7L6tQ3bcW8YksXl3IXNWasgSqf1SM/PZe6CYAYnlDtkXdsJ3yMcabOaKLJISGtOjdSw92zRh975D/O2zNcQ3qsfvzun5k6ezg+3RmekcKC7lj+f3Dumxz+jVmo7NG/GveRsY36d1yL9vker0WXo2EQandA//brqgK4+Qy9lzgAUbdnFu3zbf/7K8fWwXJpzYicnzN/LPL9eHtJ5v1u9k+tJMbj05iaSEmJAeOzLCmDiqM0u35JGSkRvSY4tUt89X7WBQh6Y0axztdSkhofAIsdkrfbeszu73w1AeZsaD5/biwgFtefKT1UxZuDkktRwsLuX301NJbNaQX57aJSTHPNKlgxNp2qgeL8zb4MnxRarD9vwiUrcVcFrPunHVAQqPkJu5PIsTEhrTvdWPh+aIiDCevKw/p/VoyQPTVzBzRVbQa5k8fyPrc/bxh/N606BeZNCPV56G0ZFcO6Ijn6bvYEPOXk9qEDlec1ftAOD0nq08riR0FB74/moIRW+n7D1FLNy4i3P6tS33/n69yAie/dkgBndoyh3/WcJXa3Mqve9DJaUBDXW+NbeQf8xdyxm9WjHW43/w147sRL3ICCbP3+hpHSJV9Xl6NonNGtK1ZWhv/XqpzodH7r6DnDdpPr+bnhr0eSY+8feyOqdvxaPPNoyOZPKEoZyQEMPPX09h6Za8n6yz90Axizbt5pWvN3L3O8s46+mv6Pn72Yx+4gsmfb6WHQVFx6zl4Y/ScDgeOq/XcX1P1SEhtj6XDGrHuylb2bX3gNfliARk/8ES5q/bydgerepUp48639uqaeNorhyayDOfr6NlkwbcFcQH5D5ekUWXljF0a3X0v07iGtbjtRuHcenz3zLh39/xfxf2YWvuflZmFrByWz4bd+3jcM41bxxN73ZxjOnWghVb8/nLnDX87bO1nN6zJVcN68CYrgk/eWDp81U7mJO2g3vGd6d905rxJOxNo5KY+t0W/v7ZWh48rxf1Iuv83zVSS3y9bicHiksZW4faO0DhAcBdZ3RjR0ER/5i7lpax9blmRMdqP4bvltVubj+ta6X+OmnZpAFv3DScS57/htumLAGgXXxDerdtwgUD2tGnXRN6t42jVZP6P9rfhpy9vLVoC++kbOWTlTtoF9+Qq4YlctmQRFo1aUDRoRIe+nAlJyQ0ZuKopGr/PquqS8sYLhvcntcXZPD1up385szu6r4rtcLcVdk0jo5keOe6NUeNhdOUoEOGDHHJyVUbvb24pJSfv57CF6uz+efVgxnfp3oflnvt2008+MFK5tw5hm6tKj+PRc6eA6zdsYeebZrQNIAugAeKS/g0bQdTFm7mm/W7iIwwxvZoSUyDKN5fvI0pE4dzYpcWVfhOgsc5x9z0bP48exVrs/cyIDGe+8/qwXBNHCU1lHOOEY/NZVCHpjx3zWCvy6kSM0txzg0JdDvdG/CLioxg0s8G0T8xntv/s4Tvqnmq1BnLs+jaMiag4ABfe8CJXVoEFBwA9aMiObdfW6bcPIIv7j6FiaM7k5KRy/uLt3F+/7Y1LjjA12X59F6tmHXHaJ64pB/b84u44oUF3PTKIlZv3+N1eSI/sTKzgB0FBzzvdOIFhUcZDaMjefn6obRv2pCJr1bfL6zsgiIWbdrNOf2OPk1rsHRu0Zj7z+rJt/eP5bUbh/HoxX09qaOyoiIjuHxoIl/cfQr3ju/Bd5t2c9bT8/jNO8vIzNvvdXki3/ssfQdmcEr3BK9LCTmFxxGaNo7mtRuH0aBeJNe//F21/LKalbodd4xeVqEQHRXBmG4JIRn4sDo0jI7kF6ecwFf3nMpNozrzwdJMTv3Llzw2K509NWgoe6m75qZnMzAxnhYx4T1feXkUHuVo37QRr944jH0Hirnu5e/IKzx4XPv7eHkW3VrF0DXAW1biE98omgfO6cXnd5/MOf3a8MK8Dfxq6pKgd60WOZodBUWs2JZfJ29ZgcKjQj3bNOGF64aweVchNx3HnBs7CopYlLGbc/q2reYK6572TRvx18sH8NC5vfhydQ7vJGsuEPHO56uyAepcF93DFB5HMfKE5vz9ygEs3pzLbVOWVOkp9Fkrsny3rPqFbqjzcHfdyE4M79yMR2akqQ1EPDM33dcV/sihhuoKhccxnN23DX84rzefpe/g9x+sDPhWyccrsujeKpYuLevmP7BgiIgwnry0P8WljvveX6HbVxJyRYd8T5Wf3rNlnX0WSeFRCdef2In/OfUEpn63mUdnplf6l9X2/CKSM3I962UVzjo0b8R9Z/Vg3poc3k7e4nU5Usd8s34nRYdKOa2OtneAwqPS7h7XnetHduTFrzby8Iy0SgXIrFTfLauzPe5lFa6uHdGREUnNeGRGOtt0+0pCaG6676nyEUnNvC7FMwqPSjIz/nB+b248qTP//tr3tPixpo39eHkWPVrH0qUOjbQZSodvX5U6x33vLdftKwkJ5xyfr8pmdNcE6kd5M5VBTaDwCICZ8ftze/LzMUm8viCDB6anVhgg39+y0lVHUCU2a8T9Z/Xgq7U7+c8i3b6S4FuZWUBWflGdmvipPLXjabEaxMy476weREUaz36xnpLSUh67uB+RR4xce3gyp7PV3hF0Vw/vyMwV2/nTx+mM6ZZAu/iGXpckYezzVdmYwal1ZK7yiujKowrMjLvHdeeOsV15O3krv3lnGSVHXIF8vMJ3y+qEEM8LXhdFRBhPXNoP5xz3vqvbVxJcc9N30L99PAmxde+p8rIUHlVkZtx5RjfuOqMb7y/Zxl1vL/3+OZCs/P2kZORyrq46QiaxWSPuP7sn89ftZMp3lZ8DftPOfUxfsi0kM0lK7ZddUMSyrfmcXsdvWYFuWx2328d2JSrSeGL2aopLHX+/YgAzV2wH1Msq1K4e3oFZqVk8+nE6Y7omkNis/Imuig6VMDt1O/9ZtJkFG3yjJ6/avof7zuoRynKlFvpi9eGnyutuF93DFB7V4JendKFeRAR/mplOSYkjq6CInm2akKRbViFlZvz5kn6c+bd53Pf+ct64afiPHuBKzyrgrUVbmLZkG/n7D5HYrCG/ObM7G3fu4/n/rmdAYhzj+yjwpWKfpWfTNq4BPVrroV+FRzW5eUwSkRHGwzPSAPjNmd09rqhuat+0EQ+c04vfTlvBmws3c+HAdny0LJP/LNrCsi15REdGcGaf1lw5NJGRSc2JiDAOFJewNnsvd7+znC4t1bVayld0qIT5a3dy6eD2dfap8rIUHtXoxlGdqRcVwfNfruf8/hoI0StXDUtkVmoW//dxGo/OTKfwYAndWsXw4Lm9uGhgu59MrFU/KpLnrh7Eec/M59Y3Upj+PyfVmmHrJXS+3bCL/YdK6uxAiEfSNLQSlrbl7efmV5Pp2y6OK4YlMjAx/ph/LX6zbifXTF7IWX3aMOlnA/XXpfzI76av4P3F21j8+zNoUC98Hg6s6jS0+vNKwlK7+IbMvGN0QNuc2KUF94zvweOzVjHgq3huHpMUpOqktnHO8Xl6NqO6tAir4Dge6qorUsbPxyQxvndrHp+9im/X7/K6HKkh0rP2kJlfpFtWZSg8RMowM568rB+dmjfiV1MXk5WvARfF92AgwKk9FB6HBS08zOxlM8s2s9Qyy540s1VmttzMpplZfAXbxpvZu/51081sZLDqFDlSbIN6/Ovawew/WMIv31zMwWI9QFjXfbYqm/6J8bSMbeB1KTVGMK88XgHGH7HsU6CPc64fsAa4v4JtnwZmO+d6AP2B9GAVKVKeLi1jefKy/izZnMcj/u7XUjdt2rmPZVvyGN9bs4GWFbTwcM7NA3YfsWyOc67Y/3YB0P7I7cysCTAGmOzf5qBzLi9YdYpU5Oy+bbjFP4LyeymaL72umrZkG2Zw4UB1vy/LyzaPG4FZ5SxPAnKAf5vZEjN7ycwaV7QTM7vFzJLNLDknJydYtUoddc+Z3RmR1IzfTlvBysx8r8uREHPOMX3pNkYmNadNnEZrLsuT8DCzB4Bi4M1yvhwFDAKec84NBPYB91W0L+fcC865Ic65IQkJCUGpV+quqMgIJv1sEE0bRXPrGynk7z/kdUkSQos355Kxq5CLBrbzupQaJ+ThYWbXA+cCV7vyn1DcCmx1zi30v38XX5iIeKJFTH2evXoQmXlF/Pb9FRryvQ55f/E2GtSL4CwNcvoTIQ0PMxsP3Auc75wrLG8d59x2YIuZHR4caiygFkvx1OCOTfn1uG58vCJLMxbWEQeKS5ixPItxvVpruJpyBLOr7lTgW6C7mW01s5uASUAs8KmZLTWz5/3rtjWzmWU2/xXwppktBwYAjwarTpHKunXMCYzq0oI/frSSNTv2eF2OBNkXq3LI33+IiwbpllV5NLaVSACy9xRx9tNf0axxNB/eNkpDVYSxn7+eTEpGHgvuP42oyPB9nrqqY1uF709EJAhaxjbgr5cPYM2Ovd8Pvy/hJ6/wIJ+vyub8/m3DOjiOh34qIgEa0y2BW08+gSkLN/Px8iyvy5EgmLE8i0Mljot1y6pCCg+RKvj1uG4MSIznvveXs2V3uX0/pBabtmQbXVvG0LttE69LqbEUHiJVUC8ygmeuGggOfjV1CYdKNP5VuMjYtY+UjFwuGtROc7ochcJDpIoSmzXi8Uv6sXRLHn/9dI3X5Ug1+X44kgG6ZXU0Cg+R43BOvzZcNawDz325nnlrNDxObeecY9qSbYzo3Jy28RqO5GgUHiLH6cFze9GtVQx3vb2U7D1FFa7nnGPjzn1MW7KVR2akMWN5ZgirlMpYvDmPjF2FaiivBD02KXKcGkZHMulngzh/0nx+/fYyXr1hGBERRu6+gyzdmsfSzXks3ZLHsq155BX6xsaKijAmz3d8uTqHhy/oTaNo/VesCaYt2arhSCpJ/2JFqkG3VrE8dF5v7n9/BVe9uIAdBUVs2uXrhWUG3VrGcmav1gzoEM+AxHiSEhrz7BfreebztSzZnMuknw2iZxv17PHSweJSDUcSAP2ERKrJlUMTWZyRy1drd9KvfRyXD01kQGI8/drHl/vL6K4zujGiczPueGspFz77NQ+d15urhiWqh49HvlidTV6hhiOpLIWHSDXxzX/eP6BtTuzSgll3jObOt5by22kr+Gb9Th67uC+xDeoFqUqpyLTF22gRU5/RXVp4XUqtoAZzEY+1iKnPqzcM457x3ZmVup1z/jGf5Vs1eWYo5Rce0nAkAdJPSaQGiIgwfnlKF966ZQTFJaVc8tw3vDx/o+YOCZEZKzI5WFKqXlYBUHiI1CBDOjVj5h2jOblbSx6ekcbNr6WwMjOf0lKFSDBNW6zhSAKlNg+RGia+UTQvXjeYf3+9icdmpfNZ+g7iGtZjaKdmjEhqxoik5vRs04TICDWsV4fNuwpJzsjlnvHd1VkhAAoPkRrIzLhxVGfO6deGb9bvZOGG3SzYsIvP0ncAENsgimGdfEEyPKkZvdo00b36KtJwJFWj8BCpwVo1acBFA9tz0cD2AGzPL2Lhxl0s2LCbhRt2MXdVNuALk2tGdOSW0Uk0bRztZcm1im84kq0ajqQKFB4itUjruAZcMKAdF/j/Ss4uKGLBxt18krqd5/+7nte+2cSEkzoxcZRCpDKWbMlj065CfnlqF69LqXWBQccuAAAPLklEQVQUHiK1WMsmDTi/f1vO79+WNTv28I+5a/nnl+t59ZsMJpzYiYmjOxPfSCFSHuccr36zifpREZzVp7XX5dQ6ukkqEia6tYpl0s8GMfuOMZzcPYFJX6xj1J+/4Kk5q8n3j6klP3h67lo+WJrJTaM666HMKrBw6kc+ZMgQl5yc7HUZIjXC6u2+K5GPV2QRWz+KG0Z15vqRHSkpdeTsPcCuvQfZtc/3+fv3ew+wa99BWsbW548X9KFdmLYDvPL1Rv7wURqXDGrPk5f2I6IO91wzsxTn3JCAt1N4iIS3VdsLePqztcxK3V7hOtGREbSIiaZ5TH2aNY4mJSOXyAjjiUv7cWbv8LqlM33JNv73raWc0asVz109qM73UlN4oPAQOZr0rAI+X5VNXMN6tIiJpkVMfZrH1Kd5TDSx9aN+9IzDpp37+NXUJazYls+EEztx31k9aFAv0sPqq8fnq3Zw82spDO3UlFduGBYW39PxUnig8BCpTgeLS/nz7FVMnr+RXm2aMOlnA0lKiPG6rCr7buNurp28kG6tYply83C1c/hVNTzq9vWaiFQoOiqC35/bi8nXDyErfz/nPjOf9xdv9bqsKlmZmc9NryyiXdOGvHLDUAVHNVB4iMhRje3Zipl3jKZPuzjuensZd729lH0Hir0uq9I27tzH9S9/R2yDKN64aTjNY+p7XVJYUHiIyDG1iWvI1JtHcMfYrkxfso3znpnPysx8r8s6pu35RVzz0kJKHbw+cbieIq9GCg8RqZTICOPOM7rx5sQR7DtYzEXPfsPbyVu8LqtCufsOcu3kheTvP8SrNwzjhFrcXlMTKTxEJCAjT2jOrDvGMDypGfe+t5zZR+kC7JW9B4qZ8MoiMnYX8uJ1Q+jbPs7rksKOwkNEAtascTQvXDuEAYnx3PGfJaRk7Pa6pB95YNoKUrflM+mqgYw8obnX5YQlhYeIVEnD6EgmXz+UtvENuenVZNbn7PW6JABmrcjig6WZ3H5aV8aF2QOONYnCQ0SqrFnjaF65YSiRZkz493fk7DngaT079x7ggemp9G0Xxy9PPcHTWsKdwkNEjkvH5o2ZPGEoOXsOcOMrizzrxuuc43fTUtlbVMxTl/enXh0fdiTY9NMVkeM2IDGeSVcNYmVmPrdNWUxxSWnIa/hwWSazV27nzjO60a1VbMiPX9cELTzM7GUzyzaz1DLLnjSzVWa23MymmVn8UbaPNLMlZjYjWDWKSPU5vVcrHrmwD1+szuH3H6QSyqGPdhQU8eAHKxnYIZ5bxiSF7Lh1WTCvPF4Bxh+x7FOgj3OuH7AGuP8o298BpAenNBEJhquHd+R/Tj2Bqd9tYdLn60JyTOcc97+/gqJDJfzlsv5E1uHh1UMpaOHhnJsH7D5i2Rzn3OEboguA9uVta2btgXOAl4JVn4gEx93junPxwHY89eka3k0J/lhY76Rs5fNV2dwzvoceBAwhL6ehvRF4q4Kv/R24B9CNS5Faxsx4/JJ+7NhTxH3vLadVk/qM7poQlGNl5u3nkY/SGN65GTec2Ckox5DyedJgbmYPAMXAm+V87Vwg2zmXUsl93WJmyWaWnJOTU82VikhVREdF8Nw1g+nSMoZfvLGYJZtzq/0YzjnufW85Jc7x5KX96/RsgF4IeXiY2fXAucDVrvwWtZOA881sE/Af4DQze6Oi/TnnXnDODXHODUlICM5fNyISuCYN6vHKDcOIa1iPy57/lme/WEdJafU1or+5cDNfrd3Jb8/uSYfmjaptv1I5IQ0PMxsP3Auc75wrLG8d59z9zrn2zrlOwJXA5865a0JYpohUk9ZxDfj49lGc2ac1T36ymitf+JYtu8v9rx+QzbsKeXRmOqO7tuDq4R2qoVIJVDC76k4FvgW6m9lWM7sJmISvHeNTM1tqZs/7121rZjODVYuIeCe+UTSTrhrI367oz6qsPZz19Fe8k7ylyl15S0sdd7+7jEgz/nxJvx9Nnyuho2loRSRktuYW8uu3l7Fw427G927Noxf3pVnj6ID2MXn+Rh6ZkcYTl/bj8iGJQaq07tA0tCJS47Vv2ogpN4/g/rN6MHfVDs78+zy+XJ19zO225xcxO3U7f569iidmr2Jsj5ZcNrjcnv4SIl521RWROigywvj5yScwqmsL7nxrKRP+vYjrRnbk/rN60jA6koKiQ6zYms/SLXks25LHsq157CjwDbhYL9IYmNiUxy7uq9tVHlN4iIgnereN48PbRvHE7NW8/PVGvlydQ71IY33Ovu/X6dyiMSee0IL+7ePonxhPzzZNaFAv0sOq5TCFh4h4pkG9SB48rxdje7bkL3NW07xxNBcOaEf/xHj6tY8jvlFg7SESOgoPEfHcSV1acFKXFl6XIQFQg7mIiARM4SEiIgFTeIiISMAUHiIiEjCFh4iIBEzhISIiAVN4iIhIwBQeIiISsLAaVdfMcoCMMovigPxyVi1veXnLWgA7q63AwFRUeyj2U9ltjrXe0b6uc6NzEww6N0dfVt656eicC3wmPedc2H4AL1R2eQXLkmta7aHYT2W3OdZ6R/u6zo3Ojc5N7T434X7b6qMAlle0rleqq56q7Key2xxrvaN9XedG5yYYdG4qf5zjEla3raqbmSW7KkySIsGnc1Nz6dzUXNV5bsL9yuN4veB1AVIhnZuaS+em5qq2c6MrDxERCZiuPEREJGAKDxERCZjCQ0REAqbwqCIza2xmKWZ2rte1yA/MrKeZPW9m75rZL7yuR37MzC40sxfN7AMzG+d1PfIDM0sys8lm9m5l1q9z4WFmL5tZtpmlHrF8vJmtNrN1ZnZfJXZ1L/B2cKqsm6rj3Djn0p1ztwKXA+ouWo2q6fxMd87dDEwArghiuXVKNZ2bDc65myp9zLrW28rMxgB7gdecc338yyKBNcAZwFZgEXAVEAk8dsQubgT64XvMvwGw0zk3IzTVh7fqODfOuWwzOx+4D5jknJsSqvrDXXWdH/92TwFvOucWh6j8sFbN5+Zd59ylxzpmVPWVXzs45+aZWacjFg8D1jnnNgCY2X+AC5xzjwE/uS1lZqcCjYFewH4zm+mcKw1q4XVAdZwb/34+BD40s48BhUc1qab/OwY8DsxScFSf6vq/E4g6Fx4VaAdsKfN+KzC8opWdcw8AmNkEfFceCo7gCejcmNkpwMVAfWBmUCsTCPD8AL8CTgfizKyLc+75YBZXxwX6f6c58CdgoJnd7w+ZCik8fKycZce8n+ece6X6S5EjBHRunHNfAl8Gqxj5iUDPzz+AfwSvHCkj0HOzC7i1sjuvcw3mFdgKJJZ53x7I9KgW+TGdm5pN56fmCuq5UXj4LAK6mllnM4sGrgQ+9Lgm8dG5qdl0fmquoJ6bOhceZjYV+BbobmZbzewm51wxcBvwCZAOvO2cW+llnXWRzk3NpvNTc3lxbupcV10RETl+de7KQ0REjp/CQ0REAqbwEBGRgCk8REQkYAoPEREJmMJDREQCpvAQz5jZ3hAc4/xKDrFfncc8xcxOrMJ2A83sJf/rCWY2qfqrC5yZdTpyqO9y1kkws9mhqkm8p/CQWs8/9HS5nHMfOuceD8IxjzYu3ClAwOEB/BZ4pkoFecw5lwNkmdlJXtcioaHwkBrBzH5jZovMbLmZ/bHM8un+GRtXmtktZZbvNbOHzWwhMNLMNpnZH81ssZmtMLMe/vW+/wvezF4xs3+Y2TdmtsHMLvUvjzCzf/qPMcPMZh7+2hE1fmlmj5rZf4E7zOw8M1toZkvM7DMza+UfFvtW4E4zW2pmo/1/lb/n//4WlfcL1sxigX7OuWXlfK2jmc31/2zmmlkH//ITzGyBf58Pl3clZ74ZLz82s2VmlmpmV/iXD/X/HJaZ2XdmFuu/wvjK/zNcXN7Vk5lFmtmTZc7Vz8t8eTpwdbknWMKPc04f+vDkA9jr/zwOeAHfKKARwAxgjP9rzfyfGwKpQHP/ewdcXmZfm4Bf+V//EnjJ/3oCvkmhAF4B3vEfoxe+uQ4ALsU3fHsE0BrIBS4tp94vgX+Wed+UH0ZpmAg85X/9B+DuMutNAUb5X3cA0svZ96nAe2Xel637I+B6/+sbgen+1zOAq/yvbz388zxiv5cAL5Z5HwdEAxuAof5lTfCNsN0IaOBf1hVI9r/uBKT6X98C/M7/uj6QDHT2v28HrPD635U+QvOhIdmlJhjn/1jifx+D75fXPOB2M7vIvzzRv3wXUAK8d8R+3vd/TsE3p0d5pjvf/CtpZtbKv2wU8I5/+XYz++Iotb5V5nV74C0za4PvF/LGCrY5Hehl9v0I2U3MLNY5t6fMOm2AnAq2H1nm+3kdeKLM8gv9r6cAfyln2xXAX8zsz8AM59xXZtYXyHLOLQJwzhWA7yoFmGRmA/D9fLuVs79xQL8yV2Zx+M7JRiAbaFvB9yBhRuEhNYEBjznn/vWjhb6JnU4HRjrnCs3sS3xT/wIUOedKjtjPAf/nEir+t32gzGs74nNl7Cvz+hngr865D/21/qGCbSLwfQ/7j7Lf/fzwvR1LpQekc86tMbPBwNnAY2Y2B9/tpfL2cSewA+jvr7monHUM3xXeJ+V8rQG+70PqALV5SE3wCXCjmcUAmFk7M2uJ76/aXH9w9ABGBOn484FL/G0frfA1eFdGHLDN//r6Msv3ALFl3s/BN7opAP6/7I+UDnSp4Djf4BtOG3xtCvP9rxfguy1Fma//iJm1BQqdc2/guzIZBKwC2prZUP86sf4OAHH4rkhKgWvxzXV9pE+AX5hZPf+23fxXLOC7UjlqrywJHwoP8Zxzbg6+2y7fmtkK4F18v3xnA1Fmthx4BN8vy2B4D9/EOanAv4CFQH4ltvsD8I6ZfQXsLLP8I+Ciww3mwO3AEH8DcxrlzNbmnFuFb2rW2CO/5t/+Bv/P4VrgDv/y/wXuMrPv8N32Kq/mvsB3ZrYUeAD4P+fcQeAK4BkzWwZ8iu+q4Z/A9Wa2AF8Q7Ctnfy8BacBif/fdf/HDVd6pwMflbCNhSEOyiwBmFuOc22u+eZy/A05yzm0PcQ13Anuccy9Vcv1GwH7nnDOzK/E1nl8Q1CKPXs884ALnXK5XNUjoqM1DxGeGmcXja/h+JNTB4fcccFkA6w/G18BtQB6+nlieMLMEfO0/Co46QlceIiISMLV5iIhIwBQeIiISMIWHiIgETOEhIiIBU3iIiEjAFB4iIhKw/wcvdu71AIthbwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "909e965800f948c1a72ec67a1d70d4bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      10.323396  9.283987  \n",
      "    1      9.148537   7.844613                                                                                         \n",
      "    2      8.142496   7.123969                                                                                         \n",
      "    3      7.453313   6.933463                                                                                         \n",
      "    4      6.891545   6.666717                                                                                         \n",
      "    5      6.49946    6.372166                                                                                         \n",
      "    6      6.170587   6.328063                                                                                         \n",
      "    7      5.906324   6.163712                                                                                         \n",
      "    8      5.729017   6.118506                                                                                         \n",
      "    9      5.567295   6.019835                                                                                         \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[6.0198355]"
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr=5e-3\n",
    "learn.fit(lr, 1, cycle_len=10, use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('focus-3')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now Jeremy had picked -4 in his notebook, so let's compare the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_reg4 = SSD_MultiHead(k, -4.)\n",
    "models = ConvnetBuilder(f_model, 0, 0, 0, custom_head=head_reg4)\n",
    "learn = ConvLearner(md, models)\n",
    "learn.opt_fn = optim.Adam\n",
    "learn.crit = ssd_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "281f7885dbcb410c9d362a39245328fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 78%|███████████████████████████████████████████████████████▏               | 49/63 [00:31<00:08,  1.56it/s, loss=91.2]"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEOCAYAAACaQSCZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd81PX9wPHXO5uEECAJe+8lewsoFhXRilat4LYodY/Wtra2/dlf/XVo7dC6WKJVcW8pWFvZexNG2CNAyB0jJED2+/fHXWjEJFzC3X3vLu/n43EP7r73+X6/7+RL7n3fzxRVxRhjjDmXKKcDMMYYEx4sYRhjjPGJJQxjjDE+sYRhjDHGJ5YwjDHG+MQShjHGGJ9YwjDGGOMTSxjGGGN8YgnDGGOMTyxhGGOM8UmM0wH4U1pamrZr187pMIwxJmysXr3ararpvpSNqITRrl07Vq1a5XQYxhgTNkRkr69lrUrKGGOMTyxhGGOM8YklDGOMMT6xhGGMMcYnljCMMcb4xBKGMcYYn1jCMMaYMJZxIJcF21xBOZclDGOMCWNvLNvLj95dH5RzBSxhiMgMEckRkYyztj8oIpkisklEnq5i37HeMjtE5PFAxWiMMeHOnV9IWv24oJwrkHcYM4GxFTeIyGhgPNBbVXsCfzp7JxGJBl4ArgB6ABNFpEcA4zTGmLDlyi8iPTk+KOcKWMJQ1QXA0bM23wv8QVULvWVyKtl1MLBDVXepahHwNp4kY4wx5izuvELS6od5wqhCF2CkiCwXkfkiMqiSMi2B/RVeZ3m3GWOMqUBVg1olFezJB2OARsBQYBDwroh0UFWtUEYq2U8r2eYpLDIZmAzQpk0bP4ZqjDGhLb+whMKSsoi9w8gCPlSPFUAZkFZJmdYVXrcCDlZ1QFWdoqoDVXVgerpPM/QaY0xEOJJfBBCxCeNj4BIAEekCxAHus8qsBDqLSHsRiQMmAJ8GNUpjjAkD7vxCAFLDvZeUiMwClgJdRSRLRCYBM4AO3q62bwO3q6qKSAsRmQ2gqiXAA8BcYAvwrqpuClScxhgTrsoTRrDuMALWhqGqE6t465ZKyh4ExlV4PRuYHaDQjDEmIri8VVJh363WGGNMYLnzPHcYjZPCvErKGGNMYLnzC2mUGEtsdHA+yi1hGGNMmPKMwQhOdRRYwjDGmLDlzi+yhGGMMebc3PmFpAWpwRssYRhjTNjyzCMVnAZvsIRhjDFh6XRRKSeLSq1KyhhjTPXKB+2lW8IwxhhTHVf5KO9kq5IyxhhTjWBPPAiWMIwxJiwFex4psIRhjDFhqXxakGDNVAuWMIwxJiy58wtJToghPiY6aOe0hGGMMWHInV8U1B5SYAnDGGPCkivI80iBJQxjjAlLnmlBgtd+AZYwjDEmLHmmBbE7DGOMMdUoLCnlREGJJQxjjDHVc2LQHljCMMaYsPPfQXvWhmGMMaYaZxJGENfCAEsYxhgTdtzeKqmIGYchIjNEJEdEMipse1JEDojIOu9jXBX7Pioim0QkQ0RmiUhCoOI0xphw48Q8UhDYO4yZwNhKtv9FVft6H7PPflNEWgIPAQNVtRcQDUwIYJzGGBNW3HlFJMVFUy8ueNOCQAAThqouAI7WcvcYoJ6IxACJwEG/BWaMMWEu2Gt5l3OiDeMBEdngrbJqdPabqnoA+BOwDzgE5Krql1UdTEQmi8gqEVnlcrkCF7UxxoQItwPTgkDwE8ZLQEegL55k8OzZBbxJZDzQHmgBJInILVUdUFWnqOpAVR2Ynp4emKiNMSaEuPMLSU0KbpdaCHLCUNXDqlqqqmXAVGBwJcXGALtV1aWqxcCHwPBgxmmMMaHMnV8U+VVSItK8wstrgYxKiu0DhopIoogI8B1gSzDiM8aYUFdSWsaxU0WOVEnFBOrAIjILuBhIE5Es4H+Ai0WkL6DAHuCH3rItgGmqOk5Vl4vI+8AaoARYC0wJVJzGGBNOjp4sQhXSgzzKGwKYMFR1YiWbp1dR9iAwrsLr/8GTYIwxxlTgcmgMBthIb2OMCSvlo7wjvg3DGGPM+XHn2R2GMcYYHzg1Uy1YwjDGmLBy5GQR8TFR1I8PWBN0lSxhGGNMGClfmtUz6iC4LGEYY0wYcTk0jxRYwjDGmLDizi9yZAwGWMIwxpiw4tTEg2AJwxhjwkZZmXL0pDPTgoAlDGOMCRvHThVRWqaOdKkFSxjGGBM2ykd5p9odhjHGmOo4tZZ3OUsYxhgTJsoTRnqyVUkZY4yphsvBeaTAEoYxxoQNd34RsdFCSr1YR85vCcMYY8KEZy1vZ6YFAUsYxhgTNo7kF5LmUPsFWMIwxpiw4c53btAeWMIwxpiw4eS0IGAJwxhjwoKqcsTuMIypnczsPG58ZSnbDuc5HYoxAXfidAlFpWWOTQsCAUwYIjJDRHJEJKPCtidF5ICIrPM+xlWxb0MReV9EtorIFhEZFqg4Tfh66ovNLN99lIdmraWguNTpcIwJKNeZQXuReYcxExhbyfa/qGpf72N2Ffv+DZijqt2APsCWAMVowtSSHW4WbndzaY+mbM3O4+k5mU6HZExAOT0tCAQwYajqAuBoTfcTkQbAKGC69zhFqnrcz+GZMKaq/HHOVlqkJPD8xH7cPqwtMxbvZl5mjtOhGRMwEZ0wqvGAiGzwVlk1quT9DoALeFVE1orINBFJCnKMJoTNychmfVYuj1zahYTYaH4+rjtdmtbnsfc2nPmjMibSuL3TgqRGYhtGFV4COgJ9gUPAs5WUiQH6Ay+paj/gJPB4VQcUkckiskpEVrlcrgCEbEJJSWkZz3yZSecm9bmufysAEmKj+duEfpwoKOZn729AVR2O0hj/c+cXESXQKLGOJAxVPayqpapaBkwFBldSLAvIUtXl3tfv40kgVR1ziqoOVNWB6enp/g/ahJT3Vmexy3WSn1zeleio/06P0L15Ax4f241/b83hjWV7HYzQmMBw5xfSOCn+G//vgy2oCUNEmld4eS2QcXYZVc0G9otIV++m7wCbgxCeCXGni0r561fb6N+mIZf2aPqt9++8sB0XdUnnqS+2WFdbE3E8g/acu7uAwHarnQUsBbqKSJaITAKeFpGNIrIBGA086i3bQkQq9ph6EHjTW64v8LtAxWnCx8wlezh8opCfje1W6eRrIsIzN/SmfnwMD81aS2GJdbU1kcOdX+Rol1rwtBcEhKpOrGTz9CrKHgTGVXi9DhgYoNBMGMo9VcxL83Ywums6QzqkVlmuSXICT1/fm0mvreLpOZn86qoeQYzSmMBx5xfSPs3Z/j820tuEhZfm7ySvsISfju12zrLf6d6U24a1Zfqi3SzYZh0hTPhT1ciukjKmKjknCjhw/LTP5bNzC3h18W7G92lB9+YNfNrnF+O607lJfX783nqOWFdbE+ZOFpVSUFzm6BgMsIRhgiwzO49xzy1k9DPz+POXmT5N6fG3f2+jTJUfX9b1nGXLJcRG89zEfuSeKuZnH1hXWxPe3A4vzVrOEoYJmi2HTjBx6jKio4TLejbluf/s4LK/LODrakZo73Tl8+6qLG4e0pbWjRNrdL7uzRvwsyu68dWWHN5blXW+4RvjmDOjvB1u9LaEYYJi08Fcbpq6jPiYKN6ZPIy/39Sft+4aQky0cOerK7n3jdUcyv12NdWzX2YSHxPFA5d0qtV57xzejgFtG/H03EzyC0vO98cwxhH/nRbE2jBMhMs4kMtNU5dTLzaatycPpZ23p8fwTmn88+GRPHZZF/6zNYcxz85n2sJdlJSWAbB+/3Fmb8zmrpEdan0rHhUl/PLK7rjzC3lp3g6//UzGBJMrvwiAdKuSMpFs/f7j3DR1GfXjY3jnh8Nom/rNboHxMdE8cEln/vXoRQxu35invtjCd/++mNV7j/HHOVtpnBTH3SPbn1cM/do0YnzfFkxduJusY6fO61jGOMGdV4gINE6yOwwTodbsO8Yt05aTkhjLOz8cWm0bRJvURGbcMYiXb+nP8VNFXPfSEpbsPMIDozuRnBB73rH8dGw3BHhmrk2DbsKPO7+QRolxxEQ7+5FtCcMExOq9R7lt+goa14/jncnDaNXo3A3WIsLYXs35148uYvKoDlzcNZ2bh7bxSzwtG9bj7pEd+GTdQdbuO+aXYxoTLKEwBgMsYZgAWLnHkyzSk+N5Z/IwWjSsV6P968fH8Itx3Zl552DiY6L9Ftc9F3ckrX48T32xxbrZmrDizi8iNcnZ9guwhGH8SFVZtN3N7TNW0DQlgXcmD6VZSoLTYZ1RPz6Gxy7rwuq9x5i9MdvpcIzxmTu/0PEutRDAuaRM3ZCTV8CSHUdYvMPNkp1HOHD8NJ2a1Oetu4fQJDl0kkW5Gwa2ZuaSPfz+n1v4TvcmJMT67w7GmEA5kl8UElVSljBMjeQVFLN811EW7XCzZKebbYfzAUipF8vwjqncc3FHru7dgpTE82+oDoToKOGXV/bglunLmblkD/dc1NHpkIypVkFxKfmFJY6P8gZLGMZH6/Yf53dfbGH1vmOUlinxMVEMbt+Y7/VvxYUd0+jRooGjC7vUxIjOaVzSrQkv/GcH1w9oFRJ/iMZUxeWdFsTpMRhgCcOcQ1FJGc/9ezsvzttB0wYJ3HtRR4Z3SqV/m0ZhXZ3zi3HduPyvC/nrV9t46poLnA7HmCr9d1oQq5IyIWzzwRP86N11bM3O44YBrfjVd3vQwA9jIkJBpybJ3DKkDf9YtpfbhrWjS9Nkp0MyplJu7yjvULgTtl5S5ltKSsv4+3+2M/6FRbjzi5h220CeuaFPxCSLcg+P6UJSfAy/m73F6VCMqdJ/55GyhGFCzI6cfK57aQl/+nKbZxDdo6MYU8n62ZGgcVIcD13SmXmZLubbQksmRJVPbZ4aAr2kLGEYAMrKlGkLd3HlcwvZd/QUf7+pH89P7Ecjh+euCbTbhrelTeNE/u+LzWcmPTQmlLjzC2mQEOPXQay1ZQnDkHXsFBOmLuOpL7YwsnM6cx8dxVW9WzgdVlDEx0Tz8yu6se1wPm8u3+d0OMZ8izu/KCQG7YE1etd5C7e7eGjWWkpKlWdv6MP3+rdEJDy6x/rL2F7NGNk5jf/9fDOp9ePqTLI04cGVXxgS7Rfg4x2GiDwsIg3EY7qIrBGRywIdnAkcVeXFeTu4fcYKmiQn8OmDI7huQKs6lyzAM+nhS7cMYECbRjz89jo+XX/Q6ZCMOcOdXxgSYzDA9yqpH6jqCeAyIB24E/hDdTuIyAwRyRGRjArbnhSRAyKyzvsYV83+0SKyVkQ+9zHGWikoLuXFeTtYssMdyNOElLyCYu55YzVPz8nkyt4t+Oj+4bRPSzr3jhGsfnwMr945iAFtG/HI22v5ZN0Bp0MyBvA0eofCtCDge8Io/9o5DnhVVddX2FaVmcDYSrb/RVX7eh+zq9n/YSDg/R2jo4TXluzh5QW7An2qkLD9cB7jX1jMV1ty+NVVPXhuQl8S46xmEiApPoaZdw5icPvGPPrOOj5aa+uAG2cVlpRyoqCE1DC7w1gtIl/iSRhzRSQZqLZLiaouAI7WJigRaQVcCUyrzf41ERsdxU2D27Jgm4vd7pOBPp2jZm88xDUvLObE6WLevGsIk0a0r5NVUNVJjIvh1TsGM7RDKj96dz0frLakYZxz9GToDNoD3xPGJOBxYJCqngJi8VRL1cYDIrLBW2XVqIoyfwV+yjmSkr9MHNyamCjhzWV7g3G6oCspLeP3s7dw35tr6NIsmc8fHMnQDqlOhxWy6sVFM/32QVzYMY3H3l/Pe6v2Ox2SqaPceeUJI7yqpIYBmap6XERuAX4J5NbifC8BHYG+wCHg2bMLiMhVQI6qrvblgCIyWURWicgql6t2g6+aNEhgbK9mvLtqP6eLSmt1jFB19GQRt81YwSsLdnHL0Da8HWJrVISqenHRTLt9ICM6pfHTDzbw7kpLGib4/juPVHjdYbwEnBKRPni++e8FXq/pyVT1sKqWqmoZMBUYXEmxC4GrRWQP8DZwiYi8Uc0xp6jqQFUdmJ6eXtOQzrh1aFtOFJTw6frIaux86vPNrNp7jGeu781T11wQEoN/wkVCbDRTbxvIyM7p/PSDDby9wsZpmOBy5YfOTLXge8IoUc+aluOBv6nq34Aaz9YmIs0rvLwWyDi7jKr+XFVbqWo7YALwH1W9pabnqqnB7RvTtWkyry/dGzHLd5aWKV9n5nBV7+bcMLC10+GEpYTYaKbcOoCLu6bz+IcbecsG95kgCqV5pMD3hJEnIj8HbgW+EJFoPO0YVRKRWcBSoKuIZInIJOBpEdkoIhuA0cCj3rItRKS6HlMBJyLcOqwtmw6eYM2+406G4jcbD+Ry7FQxF3Wp/Z2X8SSNV24dwOiu6fzy4418vTXH6ZBMHeHOKyIpLpp6caFRM+BrwrgRKMQzHiMbaAk8U90OqjpRVZuraqz3jmG6qt6qqheoam9VvVpVD3nLHlTVb43JUNV5qnpVDX+mWru2X0vqx8fwRoQ0fs/PdCECIztbwjhf8THRvHBzf7o3b8CDs9ay7XCe0yGZOiBU1vIu51PC8CaJN4EUb6N0garWuA0j1CXFx3Bd/5Z8seHQmVvBcDZ/Ww69WzWkcYRPIBgsiXExTLt9IPXiopn02sozXR6NCRR3CE0LAr5PDfJ9YAVwA/B9YLmIXB/IwJxy67C2FJWW8U6Y94o5fqqIdfuPW3WUnzVPqceUWwdw+EQh97yxmqISm+HWBIaqssd9MqR6NfpaJfUEnjEYt6vqbXh6N/0qcGE5p1OTZIZ3TOWt5fsoLQvfxu9FO9yUKZYwAqBfm0Y8c31vVuw+yq8+zoiYThImtGw6eIKDuQUh9Tfsa8KIUtWKLX1HarBv2LltWFsOHD/Nf8K4cXN+pouUerH0aZXidCgRaXzfljx4SSfeWbWf6Yt2Ox2OiUBzMrKJjhLGdA+dBcx8/dCfIyJzReQOEbkD+AJwtFdTII3p3pRmDRJ4fekep0OpFVVl/jYXIzqnERMdsXndcY+O6cIVvZrxu9lbrOeU8bs5m7IZ3K5xSLVB+jTrnKr+RESuwzOoToApqvpRQCNzUEx0FDcNacOf/7WNXa58OqTXdzqkGtmanUdOXmFI3cpGoqgo4dnv92Hfy6d4cNZaPrxvOF2a1nh4EuBJ8geOn2bb4Twys/PZdjiPna58ujVLZtKIDnRtVrvjmvC0IyefHTn53Dq0rdOhfIPP05Sq6gfABwGMJaRMGNya5/+znTeW7ePX3+3hdDg1Ur4+tSWMwEuMi2HqbQMZ/8JiJr22kk/uH3HOb4S5p4rJOJjL1uw8tmXnsS0nj+2H88kvLDlTpnlKAu1Sk/h0/UHeXZXFyM5p3DWyA6M6p9mEkXXA3E3ZAFzWM3Sqo+AcCUNE8oDKWvQEUFVtEJCoQkCT5ATG9mrOe6v389jlXcJqCvD5mS66NUumaYPQ6V0RyVo09PScunHKMu55YzVvTBpCXIynKvBkYQmbDp5gQ9Zx1mflsjHrOHuOnDqzb2pSHF2aJnP9gFZ0aZpM12b16dQkmZR6nnGxx04W8daKfby2ZA+3z1hB5yb1uWtke8b3bUlCbGgM5jL+N3dTNn1bN6R5Sj2nQ/mGaj8FVbVO3wffOrQtn60/yKfrDjJhcBunw/HJycISVu09yg8ubO90KHVKec+ph99ex31vrqFhYiwbso6zIyef8s52LVIS6N2qId8f1JoLWqbQvXmDc/axb5QUx/2jO3H3yA58vuEg0xbu5mcfbOTpOZncOqwttwxtG1L99M35O3D8NBuycnn8im5Oh/It4fO12QGD2jWiWzPP/FI3DmodFlUBS3ceobhUrTrKAeP7tmSX6yR/+/d20urH0btVQ67o1Zw+rVO4oGVD0s9jxG5cTBTf69+Ka/u1ZOmuI0xfuJu/frWdF+ft5NExXbj34o5+/EmMk+ZmeKqjLu/ZzOFIvs0SRjXK55d64qMM1uw7xoC2jZ0O6Zzmb3ORGBfNgHZVLTViAunRS7tw96gOJMVFB+QLhogwvGMawzumsdOVz/9+tpk//yuT7/VvaVWQEWLOpmy6NUsOyWWTrc/lOVzTtyXJ8TG8vjT055dSVeZty2F4x1SbxtxB9eNjgnI32jG9Pr8d34vSMmXmkj0BP58JPFdeISv3HA3JuwuwhHFOSfExXDegFbM3HsKVF9rzS+05cor9R09bdVQd0iY1kct7NuPNZXs5WaGXlQlPX205jCqM7WUJI2zdMrQtxaXKX77aRsaB3G90fwwl8zM9g8cu6tLE4UhMMN01sgMnCkpsKdkIMCcjm7apiXQL0XE31obhg05N6jO2ZzPeWr7vzAI6TZLj6ZCeRPu0+nRIS/I+T6J140RiHRpdPX+bi/ZpSbRJTXTk/MYZA9o2on+bhsxYvIdbh7UjOir0O2eYbztRUMySnW5+cGH7kO1gYwnDR3+/qR87XSfZ5cpnl/sku72PORmHOHaq+Ey5lg3r8dF9w2kS5AbIguJSlu46woRB4dH91/jX3SM7cO+ba/hyUzZXXND83DuYkPP11hyKS5XLQrT9Aixh+CwmOoquzZIrnaLh2Mkidh85ybbsPJ78bBM/fm89r905mKggftNbuecoBcVl1n5RR13WsxltGicyZeEuSxhhak5GNk2S4+nXuqHToVTJ2jD8oFFSHP3bNGLC4Db8+qqeLNzuZurCXUGNYX6mi7iYKIZ0CP2uv8b/oqOEH1zYjrX7jrN671GnwzE1dLqolHmZLi7v2SyoXzRryhKGn00c3JorejXjmbmZrNsfvLXB529zMaR947CawsT41w0DW9MgIYapC2y69XCzYLuL08WlIds7qpwlDD8TEf7wvd40SY7noVlrySsoPvdO5+nA8dNsz8m36qg6Lik+hluGtmXu5mz2HjnpdDimBuZmZNMwMZbB7UO7hsASRgCkJMbyt4n9yDp2il8GYUW2BTY7rfG6fXg7YqKEGbaoU9goKinjqy2HGdO9qWM9LH0V2tGFsUHtGvPImC58su4gH645ENBzLdjmokVKAp2ahNe6Hcb/mjZI4Oo+LXl3VRbHTxU5HY7xwbJdRzhRUMLYEO4dVS5gCUNEZohIjohkVNj2pIgcEJF13se4SvZrLSJfi8gWEdkkIg8HKsZAu390Jwa3b8yvPslglys/IOcoLi1j0XY3F3VND9m+2ya47hrZntPFpbzpHTNkQtucTdkkxkUzonOa06GcUyDvMGYCYyvZ/hdV7et9VLbMawnwY1XtDgwF7heR8FrByCs6SvjbhL7ExUTx0NtrKSop8/s51u0/Tl5hCaM6W3WU8ejevAEjO6fx2pI9Afk/Z/yntEz5ctNhRndrEhbrmwQsYajqAqDG/ftU9ZCqrvE+zwO2AC39HF7QNE+pxx+v603GgRM8M3er348/P9NFdJQwvFPofzsxwXPXyA7k5BXy6fqDTodiqrF23zHc+YVhUR0FzrRhPCAiG7xVVtXOwS0i7YB+wPJgBBYol/dsxq1D2zJ14W7meed78pf521z0b9PwzAptxgCM6pxG16bJTFu4K+CdLkztzcnIJi46itHdwmP+t2AnjJeAjkBf4BDwbFUFRaQ+njXEH1HVE9WUmywiq0Rklcvl8ne8fvPEld3p1iyZx95bT05egV+O6c4vZOOBXOsdZb5FRJg0sj1bs/NYtMPtdDimEqrKnE3ZjOicRv348Bg/FdSEoaqHVbVUVcuAqcDgysqJSCyeZPGmqn54jmNOUdWBqjowPT10PzgTYqN5fmI/8gtL+PG76ykrO/9vfQu3l3enDY9vJya4xvdtQXpyPFMXWhfbULTp4Amyjp0Om+ooCHLCEJGKk9xcC2RUUkaA6cAWVf1zsGILhs5Nk89MHfLYe+spKT2/Bsl5mS5Sk+Lo2aKBnyI0kSQ+Jpo7hrdjwTYXmdl5TodjzjJ3UzZRAmN6NHU6FJ8FslvtLGAp0FVEskRkEvC0iGwUkQ3AaOBRb9kWIlLeY+pC4Fbgkuq634arm4a04bHLuvDh2gO17jmlqrw8fyefrDvI5b1Ce+4Z46ybh7ShXmx00Oc2M+c2JyObIe1TaZwU53QoPgtYxZmqTqxk8/Qqyh4ExnmfLwIi+hPwgUs6kxAbzVNfbKGgeDUv3tzf5y51xaVl/PqTDGat2M9VvZvz66vCssexCZKGiXHcMLAVs1bs48eXdaF5Sj2nQzLA/qOn2J6Tz8TB4bUcgY30dshdIzvwf9f24uvMHCa9tpJTRedexe9EQTE/mLmSWSv288DoTjw3oV9Y9N02zpo8qgOq8Mp8u8sIFQu3ezoijAqzDiuWMBx085C2PHtDH5buPMJt01dwopqJCrOOneL6l5awdOcRnr6+N49d3tWqooxPWjVK5Lr+rXhrxT5yTvinh545P4t2uGiekkDH9CSnQ6kRSxgO+17/Vvz9pv6s23+cm6cu59jJb8//s27/ca55YQmHcgt4/QeD+f7A1g5EasLZfaM7UlqmvLLA7jKcVlqmLN5xhBGd0sJuOh9LGCFg3AXNmXLbADIP5zFhyjJceYVn3puTcYgJU5aSEBvFR/cNtxHdplbapiZxTd+WvLl87zf+f5ngyziQS+7p4rCYO+psljBCxCXdmvLqHYPYd/QUN76ylEO5p5myYCf3vrmG7s0b8PH9F9KpybeXhzXGV/eP7khRSRnTrMeUo8oHUl4Yhl/+LGGEkAs7pfH6pMG48gq59M8L+N3srYzr1ZxZdw8lrX680+GZMNchvT5X92nB60v3ciTf7jKcsmi7m+7NG4Tl37QljBAzqF1j3rx7CA0TY7nv4o48P9F6Qhn/eeCSThSUlDLdFlhyxOmiUlbvPcbIMKyOggCOwzC117tVQxb97BKnwzARqFOTZMZd0JzXluxh8qgONEwMn0FjkWD57iMUlZYxIgyro8DuMIypcx68pBMni0qZsXiP06HUOYu2u4mLiQr5tburYgnDmDqmW7MGjO3ZjFcX7yb3dNVjf4z/LdrhZmDbRmFbzWwJw5g66MHvdCKvoITXluxxOpQ6w5VXyNbsvLDsTlvOEoYxdVDPFimM6d6U6Yt2k1fNDAPGfxZ7u9OO7BRe04FUZAnDmDrqoe90Ivd0Ma8v3et0KHXCwu1uGiXGhvVyBJZEFPdNAAATrUlEQVQwjKmjerdqyMVd05m2cBcnC889+aWpPVVl0Q4XwzulhfUccJYwjKnDHrykM8dOFfPGMrvLCKSdrnwOnygM2+605SxhGFOHDWjbiJGd05i6cBeni0qdDidilU9nbgnDGBPWHvpOZ9z5Rby1Yp/ToUSsRdvdtEtNpHXjRKdDOS+WMIyp4wa1a8ywDqm8PH8nBcV2l+FvxaVlLNt1JKy705azhGGM4ZExnXHlFfLbzzc7HUrEWbvvOCeLShkRxt1py1nCMMYwpEMqPxzVgTeX7+OTdQecDieiLNrhJkpgWMdUp0M5b5YwjDEAPHZ5Vwa1a8TPP9zIjpw8p8OJGIu2u+jdqiEp9WKdDuW8WcIwxgAQGx3F8xP7Uy82mnvfWMOpIhubcb5OFBSzPis3bKczP1vAEoaIzBCRHBHJqLDtSRE5ICLrvI9xVew7VkQyRWSHiDweqBiNMd/ULCWBv03oxw5XPk98lIGqOh1SWFu68wilZRr23WnLBfIOYyYwtpLtf1HVvt7H7LPfFJFo4AXgCqAHMFFEegQwTmNMBSM6p/HId7rw0doDzFqx3+lwwtqi7W4S46Lp16aR06H4RcAShqouAI7WYtfBwA5V3aWqRcDbwHi/BmeMqdaDl3RiZOc0nvxsExkHcp0OJ2wt2uFmaIdU4mIio/bfiZ/iARHZ4K2yqizttgQqfq3J8m4zxgRJVJTw1xv70jgxjvveXGPrZtRC1rFT7Haf5MIIqY6C4CeMl4COQF/gEPBsJWUqm5mryopUEZksIqtEZJXL5fJPlMYYUuvH88LN/Th4/DQ/eW+9tWfU0JnpzCOkwRuCnDBU9bCqlqpqGTAVT/XT2bKA1hVetwIOVnPMKao6UFUHpqeH/8AYY0LJgLaNefyKbny5+TDTF+12OpywsnC7m6YN4uncpL7TofhNUBOGiDSv8PJaIKOSYiuBziLSXkTigAnAp8GIzxjzbZNGtOfynk35wz+3snpvbZol656yMmXJziNc2CkNkfCdzvxsgexWOwtYCnQVkSwRmQQ8LSIbRWQDMBp41Fu2hYjMBlDVEuABYC6wBXhXVTcFKk5jTPVEhKev70OLhvW4/821HMkvdDqkkLf50AmOniyKqOoogJhAHVhVJ1ayeXoVZQ8C4yq8ng18q8utMcYZKfViefHm/nzvxSX89vPN/HVCP6dDCmmLvO0XF3aMrIQRGX29jDEB16tlCpNHdeDjdQdZuceqpqqzaLubrk2TadIgwelQ/MoShjHGZ/eN7kiLlAT+55NNlJZZr6nKFBSXsmLP0YiYzvxsljCMMT5LjIvhF1d2Z/OhE7bgUiUO5Z7mwVlrKSopY1SXyOu1aQnDGFMjV17QnGEdUvnT3EyOnixyOpyQUFqmvLp4N2Oenc/C7S4ev6Ibo+wOwxhT14kIvxnfk/zCEv70ZabT4Tgu40Au1764mN98tpkB7Rrz5SMXcc9FHSOqO225gPWSMsZEri5Nk7ltWFtmLtnDTYPb0KtlitMhBd3JwhL+8q9tzFi8m8ZJ8Tw/sR9X9W4ekYminN1hGGNq5ZExXUhNiuPXn2RQVscawL/afJhL/zyfaYt2M2FwG/7944v4bp8WEZ0swO4wjDG1lFIvlp+O7cZP39/AR2sPcN2AVk6HFFD5hSVszMrltSV7mLMpmy5N6/PBTcMY0Lax06EFjSUMY0ytXd+/FW8t38fv/7mVy3o2JTkh/JchBSgsKWXLoTzW7z/O+qzjbMjKZacrH1WIj4niJ5d35e6RHSJm2nJfWcIwxtRaVJTwm6t7cs2Li3nu39t54srwXetsY1Yub6/cx4asXLZmn6C41FPNllY/nj6tUvhu7xb0bp1Cv9YNaZgY53C0zrCEYYw5L31aN+TGga15dfEebhzUmk5Nkp0OqcaOnyrijldXUFBcSp/WDZk0ogN9WqXQp3VDmqckRHzbhK8sYRhjzttPLu/K7I2HePLTzfxj0uCw+4D945ytHD9dzOcPjqB78wZOhxOy6lYFnDEmIFLrx/Pjy7qyaIebuZuynQ6nRlbtOcqsFfuZNKK9JYtzsIRhjPGLm4e0oVuzZH77+RZOF5U6HY5PikvLeOKjDFqkJPDwdzo7HU7Is4RhjPGLmOgonry6JweOn+bpuVvDYknX6Yt2k3k4j9+M70VSvNXQn4slDGOM3wztkMqtQ9vy6uI9/OazzSE9oG//0VP89attXNqjKZf2aOp0OGHBUqoxxq9+c3VP4mKimL5oN8dPFfHMDX2IjQ6t76aqypOfbiJKhCev7ul0OGHDEoYxxq+iooRfXtmdxklxPDM3kxMFJbx4c38SYqOdDu2MuZuy+ffWHJ4Y152WDes5HU7YCK20b4yJCCLC/aM78dQ1vfg6M4fbpq8g93Sx02EBnik+nvx0M92bN+DOC9s5HU5YsYRhjAmYW4a25bkJ/Vi7/xgTpyzDlVfodEj8+cttHM4r4P+u7UVMiFWVhTr7bRljAuq7fVow7fZB7Haf5IaXl7D/6CnHYsk4kMvMJbu5aXAb+rdp5Fgc4coShjEm4C7qks4bdw3m6Mkirn95CdsP5wU9htIy5YmPNtI4KY6fXt4t6OePBAFNGCIyQ0RyRCSjkvceExEVkUrXMRSRp0Vkk4hsEZHnJNzmGjDGfMOAto15955hqMINryxl7b5jQT3/W8v3sj4rl19d1YOUxMiYVTfYAn2HMRMYe/ZGEWkNXApUuoq8iAwHLgR6A72AQcBFAYvSGBMU3Zo14P17hpNSL5abpi7nyyBNI5JzooCn52QyolMaV/dpEZRzRqKAJgxVXQAcreStvwA/Baoa1aNAAhAHxAOxwOFAxGiMCa42qYm8f89wujRL5odvrGbawl0BHRVeVqb8z6ebKCwt47fX9Aq7iRFDSdDbMETkauCAqq6vqoyqLgW+Bg55H3NVdUuQQjTGBFh6cjxv3z2Uy3s046kvtvDrTzZRUlrm9/OUlSlPfLyRf2Zk88iYzrRPS/L7OeqSoCYMEUkEngB+fY5ynYDuQCugJXCJiIyqouxkEVklIqtcLpe/QzbGBEi9uGhevLk/PxzVgX8s28tdr68iv7DEb8cvLVMee389s1bs5/7RHbn3oo5+O3ZdFew7jI5Ae2C9iOzBkxDWiEizs8pdCyxT1XxVzQf+CQyt7ICqOkVVB6rqwPT09ACGbozxt6go4efjuvO7ay9g4XY317+0hIPHT5/3cYtLy3jknXV8uOYAP7q0Cz+5vJtVRflBUBOGqm5U1Saq2k5V2wFZQH9VPbvlax9wkYjEiEgsngZvq5IyJkLdNKQNr94xiKxjp7nmhcVkHMit9bGKSsp48K21fLb+ID8b242HbNpyvwl0t9pZwFKgq4hkicikasoOFJFp3pfvAzuBjcB6YL2qfhbIWI0xzhrVJZ0P7h1ObHQUN7y8lH9trnk/l4LiUu55YzVzNmXz66t6cO/FVg3lTxIOc9b7auDAgbpq1SqnwzDGnIecvALuem0VGw/k8vjYbtw6rC2JceeeJ/V0USmT/7GKhdvdPHVNL24Z2jYI0YY/EVmtqgN9KmsJwxgTak4XlfLIO2uZu+kwcdFRDOnQmNFdmzC6W5NKezqdLCxh0msrWb77KH+8rjffH9jagajDkyUMY0zYKytTlu46wn+25vB1Zg67XCcBaJeayOhuTRjdtQmD2zemuLSMO19dydr9x/nz9/swvm9LhyMPL5YwjDERZ++Rk8zLdPF1Zg5Ldx6hsKSMerHRNE6K4/CJAp6b2I9xFzR3OsywYwnDGBPRTheVsmzXEb7OzGHTwRPce1FHxtgyq7VSk4RhK+4ZY8JOvbhoT7VUtyZOh1Kn2PTmxhhjfGIJwxhjjE8sYRhjjPGJJQxjjDE+sYRhjDHGJ5YwjDHG+MQShjHGGJ9YwjDGGOOTiBrpLSIuYG+FTSlAZRPrV7b97G1pgNuvAfqmqpgDeQxfyp+rTE1+11VtD4Vr4MTv39d97BoE7hi+lq+uXG3eC4Vr0FZVfVt9TlUj9gFM8XX72duAVaEUcyCP4Uv5c5Wpye86lK+BE79/uwbOXwNfy1dXrjbvheo1qOoR6VVSVS26VNn2UFmgyR9x1PQYvpQ/V5ma/K6r2h4K18CJ37+v+9g1CNwxfC1fXbnavBeq16BSEVUl5U8iskp9nJDLBIZdA+fZNXBeKF2DSL/DOB9TnA7A2DUIAXYNnBcy18DuMIwxxvjE7jCMMcb4xBKGMcYYn1jCMMYY4xNLGLUkIkkislpErnI6lrpIRLqLyMsi8r6I3Ot0PHWRiFwjIlNF5BMRuczpeOoiEekgItNF5P1gnK/OJQwRmSEiOSKScdb2sSKSKSI7RORxHw71M+DdwEQZ2fxxDVR1i6reA3wfCIkuh+HET9fgY1W9G7gDuDGA4UYkP12DXao6KbCRVoitrvWSEpFRQD7wuqr28m6LBrYBlwJZwEpgIhAN/P6sQ/wA6I1nuH4C4FbVz4MTfWTwxzVQ1RwRuRp4HPi7qr4VrPgjgb+ugXe/Z4E3VXVNkMKPCH6+Bu+r6vWBjjkm0CcINaq6QETanbV5MLBDVXcBiMjbwHhV/T3wrSonERkNJAE9gNMiMltVywIaeATxxzXwHudT4FMR+QKwhFEDfvo7EOAPwD8tWdScv/4OgqnOJYwqtAT2V3idBQypqrCqPgEgInfgucOwZHH+anQNRORi4HtAPDA7oJHVHTW6BsCDwBggRUQ6qerLgQyujqjp30Eq8H9APxH5uTexBIwlDA+pZNs56+pUdab/Q6mzanQNVHUeMC9QwdRRNb0GzwHPBS6cOqmm1+AIcE/gwvmmOtfoXYUsoHWF162Agw7FUlfZNXCeXQPnhfQ1sIThsRLoLCLtRSQOmAB86nBMdY1dA+fZNXBeSF+DOpcwRGQWsBToKiJZIjJJVUuAB4C5wBbgXVXd5GSckcyugfPsGjgvHK9BnetWa4wxpnbq3B2GMcaY2rGEYYwxxieWMIwxxvjEEoYxxhifWMIwxhjjE0sYxhhjfGIJwzhGRPKDcI6rfZyu3p/nvFhEhtdiv34iMs37/A4R+bv/o6s5EWl39hTclZRJF5E5wYrJOMMShgl73imhK6Wqn6rqHwJwzurmYbsYqHHCAH4BPF+rgBymqi7gkIhc6HQsJnAsYZiQICI/EZGVIrJBRH5TYfvH3pUNN4nI5Arb80Xkf0VkOTBMRPaIyG9EZI2IbBSRbt5yZ76pi8hMEXlORJaIyC4Rud67PUpEXvSe43MRmV3+3lkxzhOR34nIfOBhEfmuiCwXkbUi8pWINPVOV30P8KiIrBORkd5v3x94f76VlX2oikgy0FtV11fyXlsR+bf3d/NvEWnj3d5RRJZ5j/m/ld2xiWdlyC9EZL2IZIjIjd7tg7y/h/UiskJEkr13Egu9v8M1ld0liUi0iDxT4Vr9sMLbHwM3V3qBTWRQVXvYw5EHkO/99zJgCp6ZOqOAz4FR3vcae/+tB2QAqd7XCny/wrH2AA96n98HTPM+vwPPAksAM4H3vOfogWfdAYDr8UyRHgU0A44B11cS7zzgxQqvG/Hf2RLuAp71Pn8SeKxCubeAEd7nbYAtlRx7NPBBhdcV4/4MuN37/AfAx97nnwMTvc/vKf99nnXc64CpFV6nAHHALmCQd1sDPDNXJwIJ3m2dgVXe5+2ADO/zycAvvc/jgVVAe+/rlsBGp/9f2SNwD5ve3ISCy7yPtd7X9fF8YC0AHhKRa73bW3u3HwFKgQ/OOs6H3n9X41krozIfq2f9ks0i0tS7bQTwnnd7toh8XU2s71R43gp4R0Sa4/kQ3l3FPmOAHiJnZq5uICLJqppXoUxzwFXF/sMq/Dz/AJ6usP0a7/O3gD9Vsu9G4E8i8kfgc1VdKCIXAIdUdSWAqp4Az90I8HcR6Yvn99ulkuNdBvSucAeWguea7AZygBZV/AwmAljCMKFAgN+r6ivf2OhZJGkMMExVT4nIPDzL4gIUqGrpWccp9P5bStX/twsrPJez/vXFyQrPnwf+rKqfemN9sop9ovD8DKerOe5p/vuznYvPE8Cp6jYRGQCMA34vIl/iqTqq7BiPAoeBPt6YCyopI3ju5OZW8l4Cnp/DRChrwzChYC7wAxGpDyAiLUWkCZ5vr8e8yaIbMDRA518EXOdty2iKp9HaFynAAe/z2ytszwOSK7z+Es8MpAB4v8GfbQvQqYrzLMEzzTV42ggWeZ8vw1PlRIX3v0FEWgCnVPUNPHcg/YGtQAsRGeQtk+xtxE/Bc+dRBtyKZx3ps80F7hWRWO++Xbx3JuC5I6m2N5UJb5YwjONU9Us8VSpLRWQj8D6eD9w5QIyIbAB+i+cDMhA+wLNwTQbwCrAcyPVhvyeB90RkIeCusP0z4NryRm/gIWCgt5F4M5WskKaqW/EsdZp89nve/e/0/h5uBR72bn8E+JGIrMBTpVVZzBcAK0RkHfAE8JSqFgE3As+LyHrgX3juDl4EbheRZXg+/E9WcrxpwGZgjber7Sv8925uNPBFJfuYCGHTmxsDiEh9Vc0XzxrJK4ALVTU7yDE8CuSp6jQfyycCp1VVRWQCngbw8QENsvp4FgDjVfWYUzGYwLI2DGM8PheRhngar38b7GTh9RJwQw3KD8DTSC3AcTw9qBwhIul42nMsWUQwu8MwxhjjE2vDMMYY4xNLGMYYY3xiCcMYY4xPLGEYY4zxiSUMY4wxPrGEYYwxxif/D1TRnyJq9102AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a24a40607de4f4caba7b8da74579b91",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 19%|█████████████▌                                                         | 12/63 [00:09<00:40,  1.26it/s, loss=15.8]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-110:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\site-packages\\tqdm\\_tqdm.py\", line 144, in run\n",
      "    for instance in self.tqdm_cls._instances:\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\_weakrefset.py\", line 60, in __iter__\n",
      "    for itemref in self.data:\n",
      "RuntimeError: Set changed size during iteration\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      11.619312  11.035986 \n",
      "    1      9.885275   8.210117                                                                                         \n",
      "    2      8.456624   7.359024                                                                                         \n",
      "    3      7.573123   7.21607                                                                                          \n",
      "    4      7.018994   6.589329                                                                                         \n",
      "    5      6.573195   6.489747                                                                                         \n",
      "    6      6.310443   6.318949                                                                                         \n",
      "    7      6.093323   6.214604                                                                                         \n",
      "    8      5.930494   6.10571                                                                                          \n",
      "    9      5.690419   6.028476                                                                                         \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[6.0284758]"
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr=1e-2\n",
    "learn.fit(lr, 1, cycle_len=10, use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('focus-4')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Very similar end losses. Now let's see how the model does when we continue the training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b9d6467195c40b0bf24733ea0cee3c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      5.627191   6.129256  \n",
      "    1      5.584963   6.024652                                                                                         \n",
      "    2      5.440576   5.920463                                                                                         \n",
      "    3      5.14999    5.875351                                                                                         \n",
      "    4      4.975074   5.703865                                                                                         \n",
      "    5      4.830699   5.639754                                                                                         \n",
      "    6      4.642337   5.639698                                                                                         \n",
      "    7      4.533329   5.578117                                                                                         \n",
      "    8      4.412708   5.581883                                                                                         \n",
      "    9      4.27575    5.523815                                                                                         \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[5.523815]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load('focus-3')\n",
    "learn.freeze_to(-2)\n",
    "lr = 5e-3\n",
    "lrs = np.array([lr/100,lr/10,lr])\n",
    "learn.fit(lrs/4, 1, cycle_len=10, use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('focus-3b')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40081430293a48a994e0fbf83528f2b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      5.906129   6.460793  \n",
      "    1      5.921797   6.342263                                                                                         \n",
      "    2      5.696199   6.043509                                                                                         \n",
      "    3      5.522027   5.826624                                                                                         \n",
      "    4      5.271962   5.80883                                                                                          \n",
      "    5      5.031068   5.731424                                                                                         \n",
      "    6      4.808868   5.759064                                                                                         \n",
      "    7      4.635631   5.612777                                                                                         \n",
      "    8      4.466427   5.528914                                                                                         \n",
      "    9      4.321626   5.497407                                                                                         \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[5.4974074]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load('focus-4')\n",
    "learn.freeze_to(-2)\n",
    "lr = 1e-2\n",
    "lrs = np.array([lr/100,lr/10,lr])\n",
    "learn.fit(lrs/4, 1, cycle_len=10, use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('focus-4b')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Again similar performances.\n",
    "\n",
    "What if we hadn't done anything? Would a model have caught up with enough training?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_reg4 = SSD_MultiHead(k, 0)\n",
    "models = ConvnetBuilder(f_model, 0, 0, 0, custom_head=head_reg4)\n",
    "learn = ConvLearner(md, models)\n",
    "learn.opt_fn = optim.Adam\n",
    "learn.crit = ssd_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d83bfa1cd41c4d3388c360c04f7dd578",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 92%|█████████████████████████████████████████████████████████████▋     | 58/63 [00:34<00:02,  1.67it/s, loss=1.61e+03]"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEOCAYAAACTqoDjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd8leX9//HXJ5sREkYIgRCCMpVNmOK2bsWBioMliLOu/vqtHd/u1vZbW0ddoChg3bgo0KIigouRsGVvwkoYgbBJcv3+ODc1xQMkISd3zsn7+Xicx7nHdZ/zySXmnXtdtznnEBEROV6U3wWIiEj1pIAQEZGgFBAiIhKUAkJERIJSQIiISFAKCBERCUoBISIiQSkgREQkKAWEiIgEpYAQEZGgYvwu4HQ0atTIZWZm+l2GiEhYycnJ2eGcSzlVu5AGhJk9BNwFGPCSc+4pM2sAvA1kAuuBm51zu83MgKeBK4EDwFDn3LyTfX5mZibZ2dkh/AlERCKPmW0oS7uQHWIysw4EwqEn0Bm42sxaA48B05xzrYFp3jzAFUBr7zUSeCFUtYmIyKmF8hxEe2CWc+6Ac64ImAFcD/QHxnltxgHXedP9gfEuYBaQbGZpIaxPREROIpQBsQQ4z8wamlltAoeOmgOpzrmtAN57Y699M2BTqe1zvWUiIuKDkJ2DcM4tM7M/A58A+4CFQNFJNrFgH/O9RmYjCRyCIiMjoxIqFRGRYEJ6matzboxzrptz7jxgF7AK2H7s0JH3nuc1zyWwh3FMOrAlyGeOds5lOeeyUlJOeRJeREQqKKQBYWaNvfcM4AbgTWAiMMRrMgT4yJueCAy2gN7AnmOHokREpOqF+j6I98ysIXAUuN+7nPVPwDtmNhzYCNzktZ1C4DzFagKXuQ4LVVHrduxn/c79tG9Sj9R68QSusBURkdJCGhDOuXODLNsJXBxkuQPuD2U9x0xetIUnPl4JQHLtWNqmJtI+rR7tmiTS1nvVjgvrewhFRE5bjfwtOLhvJj0yG7B8W6H32su72ZvYf6QYADNolVKXTunJdG6eRKf0ZNqnJRIfE+1z5SIiVadGBkS9hFh6ndGQXmc0/M+ykhJH7u6DLNu2l2Vb97Iodw8zVubx3rxcAGKjjfZp9eiUnkSPzAb0ObMhjRMT/PoRRERCzgJHdsJTVlaWC+VQG845tuw5xKJNBSzM3cPCTQUs3ryHfYcDV+u2Sa1L3zMb0ffMQNgk1YoNWS0iIpXFzHKcc1mnbKeAKJ/iEse3W/bw9ZqdfLV6B3PX7+LQ0RKiDDo2S+K8NilcelYTOjSrp5PfIlItKSCqyOGiYhZsLOCrNTv5evUO5m3cTYmDpkkJXHp2Ey49O5WemQ2IidbI6iJSPSggfLJr/xGmLdvO1G+388WqfA4XlZBcO5aL26VyRYcmnN82hViFhYj4SAFRDRw4UsTMlflM/XY705ZtZ++hIhrVjeP6rs24Kas5bVIT/S5RRGogBUQ1c7S4hBkr8nk3ZxPTluVRVOLo3DyZm7PSuaZzU+ol6AS3iFQNBUQ1tmPfYT6cv5l3s3NZsb2Q+JgoruyYxrBzMumUnux3eSIS4RQQYcA5x+LNe3gnexMfzd9C4eEi+p7ZkLvPP5PzWjfSVVAiEhIKiDBTeOgob83ZxJgv17Ft7yHaNUnknvPP5KpOaTqpLSKVSgERpo4UlTBx4RZGzVjDqrx9NEuuxYhzWzKwRwa14jTUh4icPgVEmCspcUxfkceoGWuZs34XjRPjefQHbRjQPV33VIjIaVFARJC563fx+JRlzNtYQJvUujx2RTsubNtY5yhEpELKGhD6UzQM9MhswHv39uXFO7pxtNhx59hsbn1pFgs3FfhdmohEMAVEmDAzLu+QxsePnMfv+p/Nqu376P/cV/zwzfls2nXA7/JEJAIpIMJMbHQUg/pk8vmPL+DBi1rx6dLtXPbUTF6fvYFwPlwoItWPAiJMJSbE8uilbZn2o/Pp3qI+P/9gCcPGziVv7yG/SxORCKGACHNNk2sxblhPfnPt2cxau5NLn5rJ5EVb/S5LRCKAAiICREUZQ/pmMvnBc2nRoDb3vzGPh9+az56DR/0uTUTCmAIigpyZUpcJ9/bl4Uta889FW7n8qZl8uWqH32WJSJhSQESY2OgoHr6kDe/f25dacdEMemU2z0xbRUmJTmCLSPkoICJU5+bJTP7huVzXpRl/+2QlI1/L1iEnESkXBUQEqxUXzd9u7sxvrj2bz1fkc91zX7FiW6HfZYlImFBARDizwAnsN0f2Zt/hIq5//ismLdrid1kiEgZCGhBm9oiZfWtmS8zsTTNLMLOWZjbbzFaZ2dtmFue1jffmV3vrM0NZW03TI7MBk37Yj/Zp9Xjgjfn8ccoyiopL/C5LRKqxkAWEmTUDHgSynHMdgGhgIPBn4EnnXGtgNzDc22Q4sNs51wp40msnlSi1XgJv3tWbwX1aMHrmWgaNmUPBgSN+lyUi1VSoDzHFALXMLAaoDWwFLgImeOvHAdd50/29ebz1F5uGK610cTFR/LZ/B564qTM5G3Zzy6hZuvtaRIIKWUA45zYDTwAbCQTDHiAHKHDOFXnNcoFm3nQzYJO3bZHXvmGo6qvpBnRP59VhPdi0+wADXvxGA/6JyPeE8hBTfQJ7BS2BpkAd4IogTY9doB9sb+F7F++b2Ugzyzaz7Pz8/Moqt0Y6p1Uj3rirN3sPHeXGF75m5XZd4SQi3wnlIaZLgHXOuXzn3FHgfaAvkOwdcgJIB45dUpMLNAfw1icBu47/UOfcaOdclnMuKyUlJYTl1wxdmifz9sg+ANw86hsW6BkTIuIJZUBsBHqbWW3vXMLFwFJgOjDAazME+MibnujN463/zGn86irRtkkiE+7pS72EWG5/aRZfr9bwHCIS2nMQswmcbJ4HLPa+azTwE+BRM1tN4BzDGG+TMUBDb/mjwGOhqk2+L6NhbSbc04f0+rUZOnYuH3+7ze+SRMRneia1/JeCA0cY+upcFm/ew99u7kz/Ls1OvZGIhBU9k1oqJLl2HK+P6EWPzPo88vYCPVtCpAZTQMj31ImPYcyQHnRvUZ8H35rPVB1uEqmRFBASVJ34GF4d1pNO6Uk88MY8Plu+3e+SRKSKKSDkhOrGxzB2WE/aNanHPa/NY8ZK3XciUpMoIOSkkmrF8trwnpzZuC4jx2frEliRGkQBIad07MR1ZsM6DB+XzZx137t/UUQikAJCyqRBnTj+MaIXTZMTGPbqHHI27Pa7JBEJMQWElFlKYjxv3NWblMR4hr06R0+nE4lwCggpl9R6CfxjRC8SYqMZ8socNhcc9LskEQkRBYSUW3r92oy7syf7jxQxeMxsdu/XQ4dEIpECQiqkfVo9Xh6cxabdB7lz3FwOHCk69UYiElYUEFJhvc5oyDMDu7BwUwEPvDGfo3rGtUhEUUDIabm8Qxq/u64Dny3P47H3FhPOgz+KyH+LOXUTkZO7vVcL8gsP89Snq2hcL56fXN7O75JEpBIoIKRSPHRxa/ILD/PC52toVDee4f1a+l2SiJwmBYRUCjPjt/07sHPfEX4/eSnN69fi0rOb+F2WiJwGnYOQShMdZTw1sAudmiXx8NsLWLplr98lichpUEBIpUqIjWb04CzqJcRy1/hs8gsP+12SiFSQAkIqXWq9BF4anMXO/Ye5+7VsDh0t9rskEakABYSERMf0JP52cxfmbSzgZ+/r8leRcKSAkJC5smMaj1zShvfnb+bFGWv9LkdEyklXMUlIPXhxK1bn7+P/pi7nzJQ6urJJJIxoD0JCysz4y4BOurJJJAwpICTkSl/ZNGLcXF3ZJBImQhYQZtbWzBaUeu01s4fNrIGZfWJmq7z3+l57M7NnzGy1mS0ys26hqk2q3ndXNh3h/jfmaWA/kTAQsoBwzq1wznVxznUBugMHgA+Ax4BpzrnWwDRvHuAKoLX3Ggm8EKraxB8d05P4842dmLNuF3+YvMzvckTkFKrqENPFwBrn3AagPzDOWz4OuM6b7g+MdwGzgGQzS6ui+qSKXNe1GcP7tWTs1+uZkJPrdzkichJVFRADgTe96VTn3FYA772xt7wZsKnUNrneMokwP72iHX3OaMjPPljMotwCv8sRkRMIeUCYWRxwLfDuqZoGWfa9u6vMbKSZZZtZdn5+fmWUKFUsJjqKZ2/rSkrdeO55LYcd+3TSWqQ6qoo9iCuAec657d789mOHjrz3PG95LtC81HbpwJbjP8w5N9o5l+Wcy0pJSQlh2RJKDevGM2pQ98BJ69d10lqkOqqKgLiV7w4vAUwEhnjTQ4CPSi0f7F3N1BvYc+xQlESmDs2SePyGjsxet4s/TtFJa5HqJqR3UptZbeAHwN2lFv8JeMfMhgMbgZu85VOAK4HVBK54GhbK2qR6uKFbOos37+HVr9bTsVkSN3RL97skEfGENCCccweAhsct20ngqqbj2zrg/lDWI9XTz65sz9Ite/np+4tp3TiRjulJfpckIuhOaqkGYqOjeO72bjSsE8e9r+dQcOCI3yWJCAoIqSYa1Y3nudu7sX3vIR55ewElJRoeXMRvCgipNrpm1OeXV5/F9BX5PDt9td/liNR4CgipVu7o3YLruzbjyU9XMnOl7nMR8ZMCQqoVM+MP13egTeNEHnprPpsLDvpdkkiNpYCQaqd2XAwv3NGNo8WO+16fx+EiPdNaxA8KCKmWzkipyxM3dWLhpgJ+P0k30Yn4QQEh1dblHdK4+7wzeG3WBj6Yr5FfRaqaAkKqtR9f1paeLRvw0/cXs3ybHlcqUpUUEFKtHRv5NTEhlntey2HPwaN+lyRSYyggpNprnJjAC7d3I3f3QR5+a75uohOpIgoICQtZmQ341bVnM31FPk9+utLvckRqBAWEhI07emVwS1Zz/v7Zav69ZJvf5YhEPAWEhA0z4zf9z6Zz82R+9M4CVucV+l2SSERTQEhYSYiN5sU7ulErLpqR43PYe0gnrUVCRQEhYSctqRbP396djbsO8KhGfhUJGQWEhKWeLRvwy2vO4tNleTw9bZXf5YhEJAWEhK1BvVswoHs6T09bxSdLt/tdjkjEUUBI2DIzfn9dBzqlJ/HI2wtYtV0nrUUqkwJCwlpCbDSjBnUnITaaEeOz2b1fjysVqSwKCAl7aUm1GD24O1sLDnH/G/M4Wlzid0kiEUEBIRGhW0Z9Hr+hI1+v2cnvJi31uxyRiBDjdwEileXG7ums2F7I6JlraZOayB29W/hdkkhY0x6ERJSfXN6OC9um8OuJ3/LNmp1+lyMS1soUEGb2kJnVs4AxZjbPzC4NdXEi5RUdZTxza1cyG9Xh3tdz2LjzgN8liYStsu5B3Omc2wtcCqQAw4A/nWojM0s2swlmttzMlplZHzNrYGafmNkq772+19bM7BkzW21mi8ysW4V/KqnREhNieXlwFs7BiPFzKdRwHCIVUtaAMO/9SuBV59zCUstO5mng3865dkBnYBnwGDDNOdcamObNA1wBtPZeI4EXylibyPdkNqrDC7d3Y03+fh5+awHFGo5DpNzKGhA5ZvYxgYCYamaJwEmvJTSzesB5wBgA59wR51wB0B8Y5zUbB1znTfcHxruAWUCymaWV66cRKaVvq0b86pqzmLY8j8enLPO7HJGwU9armIYDXYC1zrkDZtaAwGGmkzkDyAdeNbPOQA7wEJDqnNsK4JzbamaNvfbNgE2lts/1lm0tY40i3zO4TyZr8/fz8pfraNGwNoP6ZPpdkkjYKOseRB9ghXOuwMzuAH4B7DnFNjFAN+AF51xXYD/fHU4KJtghq+8dFzCzkWaWbWbZ+fn5ZatearT/vfosLmnfmF9N/Jbpy/P8LkckbJQ1IF4ADnh7Av8DbADGn2KbXCDXOTfbm59AIDC2Hzt05L3nlWrfvNT26cCW4z/UOTfaOZflnMtKSUkpY/lSk0VHGU8P7Er7tHo88MY8vt1yqr9tRATKHhBFzjlH4DzB0865p4HEk23gnNsGbDKztt6ii4GlwERgiLdsCPCRNz0RGOxdzdQb2HPsUJTI6aoTH8MrQ3uQVCuWO8fOZeueg36XJFLtlTUgCs3sp8AgYLKZRQOxZdjuh8DrZraIwDmMPxK4PPYHZrYK+AHfXS47BVgLrAZeAu4r808hUgap9RJ4ZVgP9h8u5s6x2ew7XOR3SSLVmgV2DE7RyKwJcBsw1zn3hZllABc45051mCmksrKyXHZ2tp8lSBiasTKfO8fO5dzWjXh5cBYx0RpQQGoWM8txzmWdql2Z/s/wDhe9DiSZ2dXAIb/DQaSizm+Twu/6d+DzFfn8+p/fUpY/kkRqorIOtXEzMAe4CbgZmG1mA0JZmEgo3dYrg7vPP4N/zNrIqJlr/S5HpFoq630QPwd6OOfyAMwsBfiUwJVJImHpJ5e1Y0vBIf70r+U0qBPHzVnNT72RSA1S1oCIOhYOnp1oJFgJc1FRxl9v6kzBgSM89t4i6teO4wdnpfpdlki1UdZf8v82s6lmNtTMhgKTCVx1JBLW4mKiePGO7nRMT+b+N+Yxe62GCBc5pqwnqX8MjAY6ERh0b7Rz7iehLEykqtSJj+HVoT1oXr8WI8Zls3TLXr9LEqkWynyYyDn3nnPuUefcI865D0JZlEhVa1AnjteG9yIxIYbBr8xhw879fpck4ruTBoSZFZrZ3iCvQjPTn1kSUZom12L88F4Ul5QwaMwc8goP+V2SiK9OGhDOuUTnXL0gr0TnXL2qKlKkqrRqXJdXh/Vkx77DDHllLnsO6mFDUnPpSiSR43RpnsyoQd1ZnVfI8LFz2a8hOaSGUkCIBHFu6xSeGdiV+ZsKGDZ2LgeOKCSk5lFAiJzAFR3TeOqWLmSv38XwsdkcPFLsd0kiVUoBIXIS13RuypO3dGH2up2MGD+XQ0cVElJzKCBETqF/l2b8ZUBnvl6zk7vGZyskpMZQQIiUwY3d0/nzjZ34YtUO7n4th8NFCgmJfAoIkTK6Oas5f7qhIzNW5nPvP+YpJCTiKSBEymFgzwz+cH0HPluex/2vz1dISERTQIiU0+29WvC7/mfz6bLt3P1ajs5JSMRSQIhUwKA+mTzuHW66U/dJSIRSQIhU0K09M3hiQGdmrd3JkFfmUHhIw3JIZFFAiJyGG7un8/TArszbWMCgMXM0dpNEFAWEyGm6pnNTnr+9G99u2cNtL81i9/4jfpckUikUECKV4LKzmzB6cBar8vYxcPQs8gsP+12SyGlTQIhUkgvbNubVoT3YuOsAt4z+hi0FB/0uSeS0KCBEKtE5rRox7s6e5O89zIAXvmZ1XqHfJYlUWEgDwszWm9liM1tgZtnesgZm9omZrfLe63vLzcyeMbPVZrbIzLqFsjaRUOnZsgFv3d2bI8WOAS9+w7yNu/0uSaRCqmIP4kLnXBfnXJY3/xgwzTnXGpjmzQNcAbT2XiOBF6qgNpGQOLtpEu/f25ekWrHc9tIspi/P87skkXLz4xBTf2CcNz0OuK7U8vEuYBaQbGZpPtQnUikyGtZmwj19adW4LiPGZ/NeTq7fJYmUS6gDwgEfm1mOmY30lqU657YCeO+NveXNgE2lts31lomErZTEeN68qze9z2jAj95dyOiZa/wuSaTMQh0Q5zjnuhE4fHS/mZ13krYWZJn7XiOzkWaWbWbZ+fn5lVWnSMgkJsTyytAeXNUpjT9OWc4fJi+lpOR7/7RFqp2QBoRzbov3ngd8APQEth87dOS9Hzs4mws0L7V5OrAlyGeOds5lOeeyUlJSQlm+SKWJj4nm7wO7MqRPC176Yh0Pv71AI8FKtReygDCzOmaWeGwauBRYAkwEhnjNhgAfedMTgcHe1Uy9gT3HDkWJRIKoKOPX157N/1zelokLt3DHy7PZpbuupRoL5R5EKvClmS0E5gCTnXP/Bv4E/MDMVgE/8OYBpgBrgdXAS8B9IaxNxBdmxn0XtOLvt3ZlYe4ebnj+K9bt2O93WSJBmXPheyw0KyvLZWdn+12GSIXkbNjFXeNzKHGOlwZn0SOzgd8lSQ1hZjmlbj04Id1JLeKT7i0a8MF9fWlQO47bX5rNRws2+12SyH9RQIj4qEXDOrx/X1+6ZCTz0FsLePazVYTzXr1EFgWEiM+Sa8fx2vCeXN+1GU98vJIfvbtQjzGVaiHG7wJEJHAZ7N9u7kxmwzo8+elK1uTvZ/Sg7qTWS/C7NKnBtAchUk2YGQ9d0poX7+jOqu2FXPP3LzXQn/hKASFSzVzeoQnv39eX+NgoBo6axbvZm069kUgIKCBEqqF2Teox8f5+9GhZnx9PWMRv/vktRcUlfpclNYwCQqSaql8njnHDenLnOS159av1DHl1jp53LVVKASFSjcVER/HLa87iLwM6MXfdbq597ku+3bLH77KkhlBAiISBm7KaB55SV1TCDc9/zfvz9GwJCT0FhEiY6JZRn0k/PJcuzZN59J2F/PKjJRwp0nkJCR0FhEgYSUmM5/URvRjRryXjv9nArS/NYvveQ36XJRFKASESZmKio/jF1Wfx7G1dWbZ1L1c98yWz1+70uyyJQAoIkTB1daemfHj/OdRLiOG2l2fz8hdrNY6TVCoFhEgYa5OayIcPnMPF7Rrz+8nLuPu1HAoO6FJYqRwKCJEwVy8hllGDuvOLq9ozfUUeVz3zJdnrd/ldlkQABYRIBDAzRpx7BhPu6Ut0lHHL6Fk8N301xSU65CQVp4AQiSCdmycz+cF+XNkxjb9MXcHgV2aTV6irnKRiFBAiESYxIZZnBnbhzzd2JGfDbq58+gtmrMz3uywJQwoIkQhkZtzSI4N/PtCPhnXiGfLKHP4weSmHi/QgIik7BYRIBGudmshHD5zDHb0zeOmLdfR/9iuWbd3rd1kSJhQQIhEuITaa31/XkVeH9mDHviP0f/YrRs9cQ4lOYMspKCBEaogL2zVm6sPnckHbFP44ZTm3vTyLzQUH/S5LqjEFhEgN0rBuPKMGdef/BnRice4eLn9qJh/O36w7sCUoBYRIDWNm3JzVnH89dB5tUxN5+O0F3P/GPPILD/tdmlQzIQ8IM4s2s/lmNsmbb2lms81slZm9bWZx3vJ4b361tz4z1LWJ1GQZDWvz9t19+PFlbfl0aR4/eHIGH8zP1d6E/EdV7EE8BCwrNf9n4EnnXGtgNzDcWz4c2O2cawU86bUTkRCKjjLuv7AVUx7qxxmN6vDI2wsZPi6brXt0bkJCHBBmlg5cBbzszRtwETDBazIOuM6b7u/N462/2GsvIiHWqnEi797Tl/+9+iy+WbOTS/82kzdmb9TeRA0X6j2Ip4D/AY499qohUOCcK/Lmc4Fm3nQzYBOAt36P115EqkB0lDG8X0umPnweHdOT+NkHi7ntpdls2Lnf79LEJyELCDO7GshzzuWUXhykqSvDutKfO9LMss0sOz9fwweIVLaMhrV5fUQvHr+hI0s27+Gyp2by3PTVerxpDRTKPYhzgGvNbD3wFoFDS08ByWYW47VJB7Z407lAcwBvfRLwvTGLnXOjnXNZzrmslJSUEJYvUnOZGbf2zOCTR8/nwraN+cvUFVz5zBfM0pPrapSQBYRz7qfOuXTnXCYwEPjMOXc7MB0Y4DUbAnzkTU/05vHWf+Z0AFTEV02SEnjhju68MjSLQ0eLGTh6Fj96ZyE79+mS2JrAj/sgfgI8amarCZxjGOMtHwM09JY/CjzmQ20iEsRF7VL55JHzuf/CM5m4cDMX/XUGb83ZqOE6IpyF8x/pWVlZLjs72+8yRGqUVdsL+fmHS5izbhfdMpL5zbUd6Jie5HdZUg5mluOcyzpVO91JLSLl0jo1kbdH9uaJmzqzYecBrn3uS34yYZHuxI5ACggRKTczY0D3dKb/+AJG9GvJ+/NzufCJzxk1Y42eORFBFBAiUmH1EmL5+VVnMfXh8+jVsgGP/2s5lz05k0+XbtdNdhFAASEip+2MlLqMGdqDscN6EB1ljBifzeBX5ujhRGFOASEileaCto3598Pn8curz2LhpgKufOYLHnprPut36G7scKSrmEQkJAoOHGHUzLW8+tU6ioodN/dozoMXtaZJUoLfpdV4Zb2KSQEhIiGVt/cQz05fzZtzNhJlxpC+mdx7/pnUrxPnd2k1lgJCRKqVTbsO8OSnK/lg/mbqxsUw/NyWDOvbkqTasX6XVuMoIESkWlq5vZC/fryCqd9uJzE+hqHnZHLnOS21R1GFFBAiUq0t3bKXZ6evYsribdSJi2Zw30xG9GtJw7rxfpcW8RQQIhIWVmwr5Nnpq5m0aAsJMdEM6tOCu849g5REBUWoKCBEJKysztvHc9NX89GCzcRER3FD12bc2a8lbVIT/S4t4iggRCQsrduxn5e/WMuEnFwOF5VwXpsURvRrybmtG6GnEFcOBYSIhLVd+4/wxuwNjPtmA/mFh2mTWpcR/c7g2i5NSYiN9ru8sKaAEJGIcLiomH8u3MrLX6xl+bZCGtWN4/ZeLbijdwudp6ggBYSIRBTnHF+v2cmYL9fx2fI84qKjuKZzU+7sl8nZTfU8ivIoa0DEnKqBiEh1YGac06oR57RqxNr8fYz9ej0TcnJ5b14uvVo24M5+LbmkfSrRUTpPUVm0ByEiYWvPwaO8M3cTY79ez+aCg2Q0qM2tPTO4sVszGtfTmE8nokNMIlJjFBWX8MnS7bz69XrmrNtFdJRxYdsUbspqzkXtGhMbrYGrS9MhJhGpMWKio7iiYxpXdExjbf4+3skOHHr6dFkejerGcUO3dG7Oak6rxnX9LjWsaA9CRCJSUXEJn6/I5+3sTXy2PI/iEkfXjGQGdE/n6k5NSapVcwcJ1CEmERFPXuEhPpi3mQk5uazK20dcTBSXnpXKgO7pnNs6pcad2FZAiIgcxznH4s17mJCTy8SFWyg4cJTGifFc360ZN3ZLr/bDeuQVHmLKoq1MWrSVBy5qxQVtG1foc3QOQkTkOGZGp/RkOqUn8/Or2vPZsjzem5fLy1+sY9SMtbRJrcvVnZpyVac0zkypHucrdu47zL+WbGPSoi3MXrcL56BtaiJFxaH/4157ECJS4+UXHuZfS7YyaeFW5m4I/BJun1aPqzulcVXHNDIb1Tnt7zh0tJjc3QfZtOsAG3cdYNOuAxT7Tuq1AAALSklEQVQcPErtuGhqx8V474HpOvHR7D9czL+WbOXrNTspLnGckVKHqzs15ZpOabQ+zT0d3w8xmVkCMBOIJ7CnMsE59yszawm8BTQA5gGDnHNHzCweGA90B3YCtzjn1p/sOxQQIlLZtu05xJTFW5m8eCs5G3YDcHbTelzSPpWL2zemQ9Mkok5xzmLf4SK+WbOTmSvzWb5tLxt3HWD73sP/1SYhNorkWnEcPFrMwSPFHCku+d7nZDSozdWd0ri6U1PapyVW2mCF1SEgDKjjnNtnZrHAl8BDwKPA+865t8zsRWChc+4FM7sP6OScu8fMBgLXO+duOdl3KCBEJJS2FBxkyuKt/GvJNuZt3I1zkJIYz0VtG3NR+8b0a9WIOvExOOdYvq2QGSvzmbEin+wNuzha7KgTF83ZTZPIaFibjAa1ad6glvdem5S68f/1C/9ocQkHjgTCYv+RIgxo2ahOSEaw9T0gjiumNoGAuBeYDDRxzhWZWR/g1865y8xsqjf9jZnFANuAFHeSAhUQIlJVdu0/wucr8pi2PI+ZK/IpPFxEXHQUXTOSWbdjP3mFgT2Edk0SOb9tChe0aUz3FvWJi6l+N+lVi5PUZhYN5ACtgOeANUCBc67Ia5ILNPOmmwGbALzw2AM0BHaEskYRkbJoUCdww90N3dI5WlzC3PW7+GxZHt+s3UmPlg04v00K57dJITWChvgIaUA454qBLmaWDHwAtA/WzHsPth/1vb0HMxsJjATIyMiopEpFRMouNjqKvmc2ou+ZjfwuJaSqZN/HOVcAfA70BpK9Q0gA6cAWbzoXaA7grU8CdgX5rNHOuSznXFZKSkqoSxcRqbFCFhBmluLtOWBmtYBLgGXAdGCA12wI8JE3PdGbx1v/2cnOP4iISGiF8hBTGjDOOw8RBbzjnJtkZkuBt8zs98B8YIzXfgzwmpmtJrDnMDCEtYmIyCmELCCcc4uArkGWrwV6Bll+CLgpVPWIiEj5VL/rr0REpFpQQIiISFAKCBERCUoBISIiQYX1aK5mlg9sIHDPxJ7jVp9q2fHrGxG6u7aD1VJZ25ys3YnWVaS/jp9Xf6m/1F/la1ed+quFc+7UN5I558L+BYwu77Lj1wPZVVlfZW1zsnYnWleR/grSf+ov9Zf6K8L7K1IOMf2zAsuCrQ+VinxXWbc5WbsTratIf53quyqT+qt81F/lo/4qo7A+xFSZzCzblWF0QwlQf5WP+qt81F/lE6r+ipQ9iMow2u8Cwoz6q3zUX+Wj/iqfkPSX9iBERCQo7UGIiEhQCggREQlKASEiIkEpIMrIzOqYWY6ZXe13LdWdmbU3sxfNbIKZ3et3PdWdmV1nZi+Z2Udmdqnf9VR3ZnaGmY0xswl+11Jdeb+vxnn/rm6v6OdEfECY2StmlmdmS45bfrmZrTCz1Wb2WBk+6ifAO6GpsvqojP5yzi1zzt0D3AxE9KWKldRfHzrn7gKGAreEsFzfVVJ/rXXODQ9tpdVPOfvuBmCC9+/q2op+Z8QHBDAWuLz0Au8hRs8BVwBnAbea2Vlm1tHMJh33amxmlwBLge1VXbwPxnKa/eVtcy3wJTCtasuvcmOphP7y/MLbLpKNpfL6q6YZSxn7jsDjnDd5zYor+oWhfKJcteCcm2lmmcct7gmsdoGHF2FmbwH9nXOPA987hGRmFwJ1CPwHOGhmU5xzJSEt3CeV0V/e50wEJprZZOCN0FXsr0r692XAn4B/OefmhbZif1XWv6+aqDx9B+QSCIkFnMaOQMQHxAk047t0hUBn9jpRY+fczwHMbCiwI1LD4STK1V9mdgGBXdx4YEpIK6ueytVfwA8JPLM9ycxaOedeDGVx1VB5/301BP4AdDWzn3pBUlOdqO+eAZ41s6s4jSE5ampAWJBlp7xj0Dk3tvJLCQvl6i/n3OfA56EqJgyUt7+eIfA/dE1V3v7aCdwTunLCStC+c87tB4ad7ofXhHMQweQCzUvNpwNbfKolHKi/ykf9VT7qr4oLad/V1ICYC7Q2s5ZmFgcMBCb6XFN1pv4qH/VX+ai/Ki6kfRfxAWFmbwLfAG3NLNfMhjvnioAHgKnAMuAd59y3ftZZXai/ykf9VT7qr4rzo+80WJ+IiAQV8XsQIiJSMQoIEREJSgEhIiJBKSBERCQoBYSIiASlgBARkaAUEFKlzGxfFXzHtWUcwr0yv/MCM+tbge26mtnL3vRQM3u28qsrPzPLPH5Y6SBtUszs31VVk1Q9BYSEJW+Y46CccxOdc38KwXeebOyyC4ByBwTwM+DvFSrIZ865fGCrmZ3jdy0SGgoI8Y2Z/djM5prZIjP7TanlH1rg6X3fmtnIUsv3mdlvzWw20MfM1pvZb8xsnpktNrN2Xrv//CVuZmPN7Bkz+9rM1prZAG95lJk9733HJDObcmzdcTV+bmZ/NLMZwENmdo2ZzTaz+Wb2qZmlekMw3wM8YmYLzOxc76/r97yfb26wX6Jmlgh0cs4tDLKuhZlN8/pmmplleMvPNLNZ3mf+NtgemQWeJjbZzBaa2RIzu8Vb3sPrh4VmNsfMEr09hS+8PpwXbC/IzKLN7C+l/lvdXWr1h0CFn1gm1ZxzTi+9quwF7PPeLwVGExiNMgqYBJznrWvgvdcClgANvXkH3Fzqs9YDP/Sm7wNe9qaHAs9602OBd73vOIvA2PkAAwgMRR4FNAF2AwOC1Ps58Hyp+fp8NwLBCOCv3vSvgf9Xqt0bQD9vOgNYFuSzLwTeKzVfuu5/AkO86TuBD73pScCt3vQ9x/rzuM+9EXip1HwSEAesBXp4y+oRGM25NpDgLWsNZHvTmcASb3ok8AtvOh7IBlp6882AxX7/u9IrNK+aOty3+O9S7zXfm69L4BfUTOBBM7veW97cW76TwJOx3jvuc9733nMIPIMimA9d4BkeS80s1VvWD3jXW77NzKafpNa3S02nA2+bWRqBX7rrTrDNJcBZZv8ZjbmemSU65wpLtUkD8k+wfZ9SP89rwP+VWn6dN/0G8ESQbRcDT5jZn4FJzrkvzKwjsNU5NxfAObcXAnsbBJ4b0IVA/7YJ8nmXAp1K7WElEfhvsg7IA5qe4GeQMKeAEL8Y8LhzbtR/LQw8bOgSoI9z7oCZfQ4keKsPOeeOf3ziYe+9mBP/ez5catqOey+L/aWm/w78zTk30av11yfYJorAz3DwJJ97kO9+tlMp86BpzrmVZtYduBJ43Mw+JnAoKNhnPELgUbqdvZoPBWljBPbUpgZZl0Dg55AIpHMQ4pepwJ1mVhfAzJpZ4HnDScBuLxzaAb1D9P1fAjd65yJSCZxkLoskYLM3PaTU8kIgsdT8xwRG2QTA+wv9eMuAVif4nq8JDN0MgWP8X3rTswgcQqLU+v9iZk2BA865fxDYw+gGLAeamlkPr02id9I9icCeRQkwCAh28n8qcK+ZxXrbtvH2PCCwx3HSq50kfCkgxBfOuY8JHCL5xswWAxMI/IL9NxBjZouA3xH4hRgK7xF42MoSYBQwG9hThu1+DbxrZl8AO0ot/ydw/bGT1MCDQJZ3UncpQZ6A5pxbTuAxo4nHr/O2H+b1wyDgIW/5w8CjZjaHwCGqYDV3BOaY2QLg58DvnXNHgFuAv5vZQuATAn/9Pw8MMbNZBH7Z7w/yeS8DS4F53qWvo/hub+1CYHKQbSQCaLhvqbHMrK5zbp8FnnE8BzjHObetimt4BCh0zr1cxva1gYPOOWdmAwmcsO4f0iJPXs9MoL9zbrdfNUjo6ByE1GSTzCyZwMnm31V1OHheAG4qR/vuBE4qG1BA4AonX5hZCoHzMQqHCKU9CBERCUrnIEREJCgFhIiIBKWAEBGRoBQQIiISlAJCRESCUkCIiEhQ/x+IqIAT3oGvYgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0fd9f9a077d342bbaff6273eae08377b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 16%|███████████▍                                                            | 10/63 [00:08<00:43,  1.22it/s, loss=249]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-41:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\site-packages\\tqdm\\_tqdm.py\", line 144, in run\n",
      "    for instance in self.tqdm_cls._instances:\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\_weakrefset.py\", line 60, in __iter__\n",
      "    for itemref in self.data:\n",
      "RuntimeError: Set changed size during iteration\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      40.559813  16.122211 \n",
      "    1      18.568694  11.207973                                                                                        \n",
      "    2      12.854556  10.102815                                                                                        \n",
      "    3      11.269436  10.136271                                                                                        \n",
      "    4      10.409943  9.105105                                                                                         \n",
      "    5      9.72097    9.010611                                                                                         \n",
      "    6      9.236041   8.544959                                                                                         \n",
      "    7      8.82257    8.325523                                                                                         \n",
      "    8      8.494837   8.004066                                                                                         \n",
      "    9      8.138645   7.663162                                                                                         \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[7.663162]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr=5e-2\n",
    "learn.fit(lr, 1, cycle_len=10, use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('focus0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee78e242c2574d6ea7cb87c43cee547f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  8%|█████▋                                                                  | 5/63 [00:05<01:08,  1.18s/it, loss=7.64]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-52:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\site-packages\\tqdm\\_tqdm.py\", line 144, in run\n",
      "    for instance in self.tqdm_cls._instances:\n",
      "  File \"C:\\Users\\Sylvain\\Anaconda3\\envs\\fastai\\lib\\_weakrefset.py\", line 60, in __iter__\n",
      "    for itemref in self.data:\n",
      "RuntimeError: Set changed size during iteration\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "  8%|█████▋                                                                  | 5/63 [00:06<01:19,  1.38s/it, loss=7.64]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      7.910938   137.57576 \n",
      "    1      7.743102   7.533659                                                                                         \n",
      "    2      7.425774   7.014266                                                                                         \n",
      "    3      7.093451   7.022715                                                                                         \n",
      "    4      6.826189   6.793766                                                                                         \n",
      "    5      6.703976   6.662138                                                                                         \n",
      "    6      6.430899   6.531081                                                                                         \n",
      "    7      6.197219   6.442986                                                                                         \n",
      "    8      6.031706   6.350862                                                                                         \n",
      "    9      5.931899   6.306448                                                                                         \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[6.3064485]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load('focus0')\n",
    "learn.freeze_to(-2)\n",
    "lr = 1e-2\n",
    "lrs = np.array([lr/100,lr/10,lr])\n",
    "learn.fit(lrs/4, 1, cycle_len=10, use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('focus0b')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We aren't even at the point we reached in just one cycle with -3 or -4 bias, so there's a clear interest in initializing them like this!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}