multifit/experiments/cls_test_wt103_1_vf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test of classifcation Moses + Fastai pre/post processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "%cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.text import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ulmfit.train_clas import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch size: 70\n",
      "Max vocab: 60000\n",
      "Cache dir: data/imdb/models/vf60k\n",
      "Model dir: data/imdb/models/vf60k/lstm_None.m\n"
     ]
    }
   ],
   "source": [
    "exp = CLSHyperParams('data/imdb', qrnn=False,tokenizer='vf', lang='en', cuda_id=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp.pretrained_model = URLs.WT103_1\n",
    "exp.drop_mult=0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp.load_cls_data??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving tokenized: cls.trn 25000, cls.val 25000\n",
      "Size of vocabulary: 60002\n",
      "First 20 words in vocab: ['xxunk', 'xxpad', 'xxmaj', 'the', ',', '.', 'and', 'a', 'of', 'to', 'is', 'it', 'in', 'i', 'this', 'that', '\"', \"'s\", 'was', 'as']\n"
     ]
    }
   ],
   "source": [
    "data_clas, data_lm = exp.load_cls_data(bs=40,force=True, use_test_for_validation=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training args:  {'drop_mult': 0.3, 'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': 'https://s3.amazonaws.com/fast-ai-modelzoo/wt103-1'} dps:  None\n"
     ]
    }
   ],
   "source": [
    "learn = exp.create_lm_learner(data_lm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.true_wd=True\n",
    "learn.opt=None\n",
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEKCAYAAAAvlUMdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xd8VtX9wPHPN4MEyIIkIBAgTBmyI4pMtVpFBAVnpXXWDrXa1lrpsIq1zipVa1t+1FFttYijigOUIeCChAjIDEsSRgYJIYPs7++P5wYDJCRA7rPyfb9ez4v73Hvuc7+HB/LNOefec0RVMcYYY5pbiK8DMMYYE5wswRhjjHGFJRhjjDGusARjjDHGFZZgjDHGuMISjDHGGFdYgjHGGOMKSzDGGGNcYQnGGGOMK8J8HUBzSUhI0OTkZF+HYYwxASUtLS1PVRPd+OygSTDJycmkpqb6OgxjjAkoIvKNW59tXWTGGGNcYQnGGGOMKyzBGGOMcYUlGGOMMa6wBGOMMcYVlmCMMca4whKMMcYYV1iCMcaYAPZGWhavrtzl6zDqZQnGGGMC2Ly0LN5cneXrMOplCcYYYwJYQWkFcW1a+TqMelmCMcaYAJZfUkF7SzDGGGOak6pyoLSSuLbhvg6lXpZgjDEmQJVUVFNRXWMtGGOMMc2roKQCgHZtLcEYY4xpRgWlToKxFowxxpjmVFBaCUB7G4MxxhjTnGq7yOw2ZWOMMc2qtovMBvmNMcY0q4KSCkQgprV1kRljjGlG+aUVxLUOJzREfB1KvVxPMCISKiLpIjK/nmPdRWSRiKwVkaUiknTU8RgR2S0iz7odpzHGBJqC0kq/vYMMvNOCuRPY2MCxJ4B/qepgYCbw8FHHHwQ+cTE2Y4wJWAUlFX77DAy4nGCcFsklwJwGigwAFjnbS4Apdc4dAXQEFroZozHGBCpPC8Y/x1/A/RbMLOAeoKaB42uAac725UC0iMSLSAjwZ+BXLsdnjDEBq6CkomV2kYnIJCBHVdOOU+xuYLyIpAPjgd1AFfBT4H1VzWzkGreKSKqIpObm5jZX6MYY4/dUlYLSCtr7cRdZmIufPRqYLCITgUggRkReUdXptQVUdQ8wFUBEooBpqlooIqOAsSLyUyAKaCUixap6b90LqOpsYDZASkqKulgXY4zxK4cqqymvqvHbhyzBxQSjqjOAGQAiMgG4u25ycfYnAPmqWuOUfd4597o6ZW4AUo5OLsYY05L5+zQx4IPnYERkpohMdt5OADaLyBY8A/oPeTseY4wJRP4+TQy420V2mKouBZY62/fV2T8PmNfIuS8CL7oWnDHGBKB8J8H48xiMPclvjDEB6Nup+q2LzBhjTDM6vNiYH3eRWYIxxpgAVDvIH+unE12CJRhjjAlIBaUVxLYOJyzUf3+M+29kxhhjGlRQWunXA/xgCcYYYwJSQUkFcX48wA+WYIwxJiAVlFb47UqWtSzBGGNMAPK0YCzBGGOMaWb5pRV+PU0MWIIxxpiAc6iimrJK/57oEizBGGNMwKl9it/uIgsCt/9nNU8vyvB1GMYYAwTGNDHgpckuA9mGPQeZv3YvrUKzmTYiiS5xrX0dkjGmhSso8TzF78/TxIC1YBr131W7aBXm+Wt6xloxxhg/cLgFY11kgausspq30ndz8Rmn8b2zuvF6WhY78kp8HZYxpoX7tovMEozfq6iqqXf/++v2crCsiqvP7Mpt5/amVWgIsz7e4uXojDHmSLVdZPYkv5/LzC/lu7OW8dGG7GOOvbYyk+T4NozqGU9idAQ3jE7mnTV72LyvyAeRGmOMR0FpBdGRYYT78USXYAmGxOgIoiLC+MXcr9i1v/Tw/q05xazcmc/VZ3ZDRAD40bieRLUK48mPNvsqXGOMIb+kwu9vUQYvJBgRCRWRdBGZX8+x7iKySETWishSEUly9g8Vkc9FZL1z7Gq34osMD+W564YTIsJP/p1GWWU1AHNTMwkLEa4YkXS4bFybVtwyticL1mezNuuAWyEZY8xxFZT6/zQx4J0WzJ3AxgaOPQH8S1UHAzOBh539pcAPVHUgcBEwS0Ti3Aqwa/s2PHX1ENbvOcgD766noqqGN9Ky+E7/jiRGRxxR9qYxybRrE84zi7e6FY4xxhyXZ6JL/x5/AZcTjNMiuQSY00CRAcAiZ3sJMAVAVbeoaoazvQfIARLdjPW8fh257dxevLoyk7v+m87+kgquHtn1mHLRkeFMGdqF5Rm5lFdVuxmSMcbUq6Ck0u/vIAP3WzCzgHuA+m/TgjXANGf7ciBaROLrFhCRkUArYNvRJ4vIrSKSKiKpubm5pxzsz7/Tl1E943l/3T66xLVmXJ/6c9rYPgmUVdaQtrPglK9pjDEnqqC0wu+fgQEXE4yITAJyVDXtOMXuBsaLSDowHtgNVNX5jE7Ay8CNqnpMklLV2aqaoqopiYmn3sAJCw3h6WuHcXrHaH4yoRehIVJvubN7xhMeKizLyDvlaxpjzIkoq6ymtKLa76eJAXenihkNTBaRiUAkECMir6jq9NoCTvfXVAARiQKmqWqh8z4GeA/4nap+4WKcR0iMjuDDu8YevnOsPm0jwhjWrR3LM3K59+J+3grNGGM4UOpME9OSWzCqOkNVk1Q1GbgGWFw3uQCISIKI1MYwA3je2d8KeAvPDQCvuxVjQ46XXGqN65PA+j0H2V9c7oWIjDHG4/BMyjYGcywRmSkik523E4DNIrIF6Ag85Oy/ChgH3CAiXzmvod6O9XjGOuMzK7ZaN5kxxnsKSjwJJhBuU/bKbMqquhRY6mzfV2f/PGBePeVfAV7xRmwn64wuscS2Dmd5Rh5ThnbxdTjGmBaiwOkiswctg1hoiDCmdwLLM3JRVV+HY4xpIfIDZC0YsARzSsb2SSD7YDlbc4p9HYoxpoUIpC4ySzCnYEyfBAC7XdkY4zUFpRVERYQdXqfKn/l/hH4sqV0beia0ZXnGkQ95frl9P498sImsgtIGzjTGmJNTUFJBu7b+3z0GtmTyKRvbJ4G5qVmUV1UTERbK++v2ctdrX1FRXcOc5duZOrwLP53Qm+SEthworWDhhmzeX7eXjXsPMu/H59C1fRtfV8EYE0AKSgNjmhiwBHPKxvZJ5KXPvyHtmwK25ZZw3/++ZkS3dvzx8jN4bWUmr67cxby0LAYlxbF+dyFVNUqXuNZkHyxnwfp93DK2p6+rYIwJIAWlFZZgWoqze8UTFiL87u2v2Z5bwnf6d+CZa4fTulUo908eyE/P7cX/LdvOF9vzuWVsTyYOOo1BXWL5zpOf8MmWXEswxpgTsr+4gl6JUb4Oo0kswZyiqIgwhndrx8qd+Vw5IomHpw4irM4qcx2iI/ntJQOOOW983w78+8tvKKusJjI81JshG2MClKqSW1x+zDIi/soG+ZvBvRP78afLB/HYFYOPSC7HM65vAuVVNXyxfb/L0RljgkVReRUVVTUkRlmCaTGGd2vH987q1qQ5zGqd3TOeiLAQlm2xW5yNMU2TV+SZ+zAhOjDGYCzB+EhkeCgje7Tnky05vg7FGBMgcmsTjLVgTGPG901kW26JPS9jjGmSvGLPU/w2BmMaNb6vZ0Zm6yYzxjRFblEZYC0Y0wS9O0TRKTaSZVtOfblnY0zwyyuuIEQImOdgLMH4kIgwvm8in27No7L6mBWhjTHmCHnF5cRHRTS4nLu/sQTjY+P6JlJUXsVXmQd8HYoxxs/lFpUHTPcYWILxudG9EwgNEesmM8Y0Ki+AHrIELyQYEQkVkXQRmV/Pse4iskhE1orIUhFJqnPsehHJcF7Xux2nr8S2Dmdo1zg+sQRjjGmEpwUTGOMv4J0WzJ3AxgaOPQH8S1UHAzOBhwFEpD3wB+AsYCTwBxFp54VYfWJ830TW7S5kf3G5r0MxxvgpVSWvuCJgnuIHlxOM0yK5BJjTQJEBwCJnewkwxdn+LvCRquaragHwEXCRm7H60vi+iajC0s3WijHG1O9gWRUV1TXWRVbHLOAeoKFbpNYA05zty4FoEYkHugCZdcplOfuC0uCkWLrEtebdtXt8HcoJqayuQVV9HYYxLUKgPcUPLiYYEZkE5Khq2nGK3Q2MF5F0YDywG6gC6rsH75ifZCJyq4ikikhqbm7g/vYvIkwa0okVGXnkO+tt+7MDpRU8uXAzw2d+xI9fSaOiym6xNsZteU4XurVgPEYDk0VkJ/AacJ6IvFK3gKruUdWpqjoM+K2zrxBPi6VrnaJJwDG/3qvqbFVNUdWUxMREl6rhHZOHdKaqRvng671euV5haSX/XbWLwkOVTT6nNrGMfXQJTy/eSv/OMSxYn81PXkmjvKraxWiNMbUJxlowgKrOUNUkVU0GrgEWq+r0umVEJEFEamOYATzvbC8ALhSRds7g/oXOvqA1oFMMPRPb8s5X3ukme2D+en79xjrGPrqYv3ycwcGy4yealTvymfDEUp5evJWxfRP48K6xzP3RKB687AwWbcrhxy+nUVZpScYYt3zbRWZ3kTVIRGaKyGTn7QRgs4hsAToCDwGoaj7wILDKec109gUtEWHykM6s3JnPvsIyV6+1NusAb67ezdThXTi7ZzxPfbyFMY8s5ulFGRTVk2g+/Hof0//5Je3btuKDO8fy3HUj6HdaDADfP7s7D08dxJLNudxqScYY1+QVlxMaIgEzTQx4KcGo6lJVneRs36eq7zjb81S1j6r2VdVbVLW8zjnPq2pv5/WCN+L0tUuHdEYV5rs42K+q/HH+RuLbtuKByQOZ/YMU5t8xhrN7xvPkR1sY+9gS/rpkK8XlVQC88sU3/PTfaQzsHMO8H59D/04xx3zmtSO78di0wSzPyOXXb6x1LXZjWrLconLi27YiJECmiQFbMtmv9EqMYmDnGN5du5dbxvZ05Roffr2PlTvzeejyM4iODAfgjC6xzP5BCl/vLmTWx1t4fMFm5izfzjm9E3hv7V7O79eBZ783nNatGl7a+aozu5J14BBPL8rgurO6M7JHe1fiN6alyiuuCKjxF7CpYvzOpUM6sybzAN/sL2n2zy6vqubhDzZxesdork7peszxM7rEMuf6M/nfbaMZ0jWO99bu5eqUrvzj+yOOm1xq/WR8LzrFRjJz/npqauz2ZWOaU6BNEwOWYPzOpMGdAJi/tvnvJnvps53syi/ld5P6Exba8Fc/pGscL944kpW/OZ9Hpg06btm6WrcK5d6L+/H17oPMW53VXGG3OFXVNRQEwO3qxrsCbaJLsATjd5LatWFE93andDdZZXUNzy7O4Pv//JLfvLWO2cu28d7avTyzaCvnnp7I2D5Nu6W7Q0wkIifW3zt5SGeGdYvj8QWbD4/j1CqrrD58q6Vp2K/mreXshxfxzprAevDWuEdV2V9cYS0Yc+omD+nM5uwiNu8rOuFzt+UWc8XfPuOJhVvIPljGB+v28qf3N3Hbf1ZzqLKa317S34WIvyUi/OHSgeQWlfPckq0A1NQob67OYvzjSxj9yGL+uWKHdaE1YNXOfN5K303biDB+9mo6jy/YZH9XhoOHPNPEBNItymCD/H5p4qBOPPTeRp5YuJnZ3x/RpFaEqvLKF9/w0PsbiQwP5a/fG84lTndbYWkl3+SXEBYSQu8O0W6Hz9CucUwd1oU5K3YwoHMM/7d8B2syDzA4KZb+nWJ4cP4GFq7fxxNXDqFr+zauxxMoqmuU+99ZT6fYSD68axwPv7+Rvy7ZxuZ9RTx19dDDN2WYlie32PPogrVgzClLjI7gnotO56MN2fz7y12Nli+vquYnr6zm9/9bz8ge8Sy4a9zh5AIQ2yacwUlxDOh87C3Gbrnnon6EinD7f9LZe+AQf75yCG//dDQv3HAmj10xmA17DvLdWcuYuyqz8Q/zYxnZRfzs1XTSvjn1x7Tmpmayfs9B7r24H7Gtw3l46iAemDyQJZtzufSZFTy/Yod1MbZQuUWeMblAmkkZrAXjt24a3YNlGXk8OH8DI3u0p2/H+lsehyqqufXlVJZn5PHbif25ZWyPEx43ccNpsZE8ceUQdu4v4YZzkmkb8e0/tatSujK6dwK/en0N97yxlgGdYzijS6wPoz1xNTXKi5/t5JEPN1FRVcOijdm8dNNIUpKPvT07M7+UA6WVxEe1on3bVkSGH3tHXuGhSp5YsJmU7u2YPKQz4OluvP6cZPp0jOLh9zcxc/4GHnp/IxP6JnL58C6M7ZNIbGtr1bQEubXTxARYC0aCZTbclJQUTU1N9XUYzSq3qJyL/7KMhKgI3r5t9DE/mIrLq7jpxVWs2pnPo1MHc9WZx9567M8OllUy5pHFjOwRz5zrU3wdTpPtLTzEr15fy4qteZzfrwM/v6Avd7yaTs7BsiOSzKGKamYt2sKc5TuorjOOEh0RxtBucVw/Kpnz+nUgJER4cP4Gnv90B+/ePqbBZLslu4g3Vmfxdvpusg+WEyKeO/7G9k5gTJ9EhnWLI7yJd/yZwPL8ih3MnL+B1b+/gPZtm3ccRkTSVNWV/4CWYPzc0s053PDCKq4f1Z0HppxxeH9haSXXv7CSdbsLefKqIUwZGpirGTy9KIMnP9rCu7ePYVCS/7diMvNLmfTMCiqra/j9pAFcc2ZXRITsg2VcM/sLcg6W8a+bR1JWWcNv3lrHN/tLuSolifP7dyS/pIL9xeXkFpWzcEM2ewvL6B7fhsuHdeHZxVu5YkQSj0wb3GgM1TVK2jcFLM/IZXlGHmuzDlCjnsQ1uncC409PZHzfRDrHtfbC34jxhsc+3MTsZdvZ8seLm/1JfkswTRCsCQZg5rue325H946nqKyKgtIK8ooqqK5RnvneML478DRfh3jSisoqGfPoElK6t+OfN5zp63Aa9fAHG5mzfAcL7hpH7w5RRxyrTTK7DxyioqqG5Pg2/GnqIM7plXDM51RW1/Dh1/t44dMdrN51gOiIMJb8asJJPedQWFrJZ9vyWJaRy9LNuex15rI7LSaSAZ1jGOi8zu3XgYiwxh+YNf7nV6+vYVlGLl/+5jvN/tluJhgbgwkAv774dHKLy9m1v4R2bVrRKzGKuDbhTBzUiTPr6fMPJNGR4fxwbA+eWLiFNZkHGNI1ztchNaiyuoY30nZzXr8OxyQXgI4xkbx269nc+Vo6w7u142fn96l3vAUgPDSES4d05tIhnVmXVUhoiJz0Q3SxbcK5eFAnLh7UCVUlI6eYZVtyWb/nIOv3FPLJllyqa5SLBp7G36YP94sxOnNiAvEpfrAEExAiwkJ55tphvg7DNdefk8ycFTuY9fEWXrhxpK/DadDSzbnkFZfXO81OLU+SGXVCn9ucXYMiQt+O0UfcFFJWWc3fP9nGrI8zeHP1bqaNSGq26xnvyC0OvKf4wW5TNn7A04rpyZLNuaTvKvB1OA2am5pJYnQEE04PrMXtIsNDueO8PoxMbs8f3llPZn6pr0MyJyivKPAmugRLMMZPXH9OMu3btmLWxxm+DqVeOUVlLN6Uw9ThXZo8N5s/CQ0R/nzVEFSVu19fY7MDBJCaGmV/SWB2kQXe/xQTlKIiwvjRuJ58siWX11P97+HLt1bvprpGueo43WP+rmv7Nvxh8kC+3JHPP1fs8HU4pokKD1VSWa3B24IRkV4iEuFsTxCRn4mI/47GmoB005gejOmdwIw31/Hp1jxfh3OYqjI3NZOU7u3olXjs4H4guXJEEhcO6MjjCzazad9BX4djmqB29oZgbsG8AVSLSG/gn0AP4D+uRWVapPDQEJ6bPpxeiVH8+OU0tmSf+GSfbli9q4BtuSUB3XqpJSI8PHUQMa3DuGfe2iMeADX+6fBT/AE20SU0PcHUqGoVcDkwS1V/DnRq5BwARCRURNJFZH49x7qJyBLn+FoRmejsDxeRl0RknYhsFJEZTa2QCWwxkeE8f+OZRLYK5cYXVpFTVObrkJi7Kos2rUKPmN8tkMVHRfD7SQNYm1XIf778xtfhmEbkFjktmGDtIgMqReRa4HqgNlE0dRKkO4GNDRz7HTBXVYcB1wDPOfuvBCJUdRAwAviRiCQ38XomwHWJa80LN5xJQWkFN7+Yyq79vrvrqaS8ivlr9zBpcKcj5lMLdJOHdGZ073ge+3CzXyRx07C8YmeiyyDuIrsRGAU8pKo7RKQH8EpjJ4lIEnAJMKeBIgrUTvEbC+yps7+tiIQBrYEKwDqMW5AzusTyzLXD2LyviAlPLOG2f6/mq8wDXo/j1ZW7KKmoDorusbpEhAennEF5VQ0PvdfQ73/GH+QWlRMeKgE5sWmTEoyqblDVn6nqqyLSDohW1UeacOos4B6gpoHj9wPTRSQLeB+4w9k/DygB9gK7gCdU9dTnQzcB5fz+HVn+63O5dVwvlmXkctlfP+Wqf3zOngOHmnS+qnLzi6uY+JflzF2VSVll9Qldf19hGU99tIUJpycyonu7k6mCX+uZGMWPJ/Tif1/t8aubKsyR8orLiW8bEZAzMDT1LrKlIhIjIu2BNcALIvJkI+dMAnJUNe04xa4FXlTVJGAi8LKIhAAjgWqgM54bCn4pIj3rucatIpIqIqm5ublNqYoJMB1jIrn34n58PuN8fj9pAF/vLuSP721o0rlLNuewaFMOecXl3PPGWkY/spg/L9xMzsGmdQn98b0NVNYoD0weGJD/uZvipxN60T2+Db9/+2vKq04sARvvCNRpYqDpXWSxqnoQmAq8oKojgMZmXRsNTBaRncBrwHkicnS32s3AXABV/RyIBBKA7wEfqmqlquYAnwLHTMamqrNVNUVVUxITA+vpanNioiLCuHlMD24d15P31+1jdSNP/NfUKI8v2EL3+Das+PV5/OeWsxjWrR3PLtnK6EcX84v/fsXXuwsbPH95Ri7z1+7ltgm96R7ftrmr4zciw0OZOeUMtueV8NySbb4Ox9Qjt6g8IO8gg6YnmDAR6QRcxbeD/MelqjNUNUlVk/EM4C9W1elHFdsFnA8gIv3xJJhcZ/954tEWOBvY1MRYTRD74dieJERF8PD7GzneTODz1+1l496D/OKCvrQKC+Gc3gnMuT6FJb+cwHVndefD9fuY9MwKrv7H53y0IfuIJ9vLq6q573/rSY5vw4/GH9NwDjrj+yZy2dDOPLM4g8+sq8zvtIQWzExgAbBNVVc53VUnNaeHiMwUkcnO218CPxSRNcCrwA3q+anxVyAK+BpYhafVtPZkrmeCS9uIMH5+QR9W7Szgow3Z9ZaprK7hyYWb6XdaNJcO7nzEseSEttw/eSCfzzif30zsR1bBIX74r1Qu+ssy3k7fTVV1Df+3bDs78kqYOeWMBmdDDjZ/vHwQPROjuOPVdPYWNm2My7ivsrqG3KJyOsZE+jqUk2LrwZiAU1Vdw4WzliHAgrvGHTM32KsrdzHjzXXM+UEK3xnQsdHPmr92L88t3cqW7GK6tW9D9sEyzu/fgeeuG+FiLfzP1pxipjy7gr6nRfPfW0fRKsxmkvK1b/aXMP7xpTx2xWDX7mR0cz2Ypg7yJ4nIWyKSIyLZIvKGcwuyMV4XFhrCvRf1Y1tuCXNTs444VlZZzV8+zmB4tzjO79+hSZ912bAufHjnOGZ/fwTt2oTTulUov580wK3w/VbvDlE8ceUQ0ncdaPKNFMZdu5yZr7u1b+PjSE5OU58cewHP1DBXOu+nO/sucCMoYxpzwYCOnJncjqc+3sI5veIJEaFalbfTd7PvYBlPXT30hO78CgkRLhx4GhcM6Eh1jQbkjMnN4eJBnbh1XE9mL9vO0K5xTB1uv0f6Uma+p7sy2BNMoqq+UOf9iyJylxsBGdMUIsK9F/dn2t8+Y8ITS484NrZPAqN6xZ/054aFBuctyU11z3dPZ03mAe7733rG9EmgQ3Rg9v8Hg135pYSHSsCOwTQ1weSJyHQ8A/HgeX5lvzshGdM0I7q344UbzyTnYBmhISGEhXiSw7mnN941ZhoWFhrCI9MGc+FTn/D4h5t5/Mohvg6pxcrMLyWpXRtCQwLzl56mJpibgGeBp/BM4/IZnuljjPEpSybu6JHQlpvG9OAfn2znurO7M7Srrc7hC7vyS+kaoN1j0PSpYnap6mRVTVTVDqp6GZ6HLo0xQeqO8/qQGB3B/e+stxUwfWRXfind2rf2dRgn7VRGMn/RbFEYY/xOVEQY917Uj68yD/Bm+m5fh9PiFB6qpPBQZcAO8MOpJZjA7BQ0xjTZ5cO6MKxbHI98sImiskpfh9OiZDq3KHdt1zITjLWZjQlyISHC/ZcOJK+4nGcWb/V1OC3K4QQTwC2Y4w7yi0gR9ScSwbNOizEmyA3pGsc1Z3Zl9rLthIUIv7zw9IC9qymQHH7IMj5IE4yqRnsrEGOM/3pgykBCQoTnlm5j3e5Cnr5mGO3aBuYMv4FiV34pcW3CiYkMvIXGarXMx5WNMSckIiyUP10+iEenDeLL7flc+uyK4y53YE5dZsGhgB7gB0swxpgTcPWZ3Xj9x6OorlGu+Ptnja7LY05eZn5pQA/wgyUYY8wJGtI1jnduH8NpMZHc/OIqtucW+zqkoFNdo2QVBPZDlmAJxhhzEhKjI3jpppGEiHD9CyvJKWraMtSmafYdLKOyWq2LzBjTMnWPb8vzN5xJXlEFN7+YSkl5la9DChq79gf2NP21LMEYY07akK5x/PW6YWzYe5Cf/ns1VdU1vg4pKGQWWIIxxhjO69eRB6ecwSdbcnn+0x2+DicoZOaXEiLQKS4wp+mv5XqCEZFQEUkXkfn1HOsmIkuc42tFZGKdY4NF5HMRWS8i60QksP+mjQli147sygUDOvLkR1v4Zn+Jr8MJeLvyS+kc15rwAF/4zhvR3wlsbODY74C5qjoMuAZ4DkBEwoBXgB+r6kBgAmATIRnjp0SEB6ecQXhICDPeXIeqzSR1KjyzKAd29xi4nGBEJAm4BJjTQBEFYpztWGCPs30hsFZV1wCo6n5VrXYzVmPMqTktNpJfX9yPz7bt5/W0LF+HE9AyLcE0ySzgHqChkb/7gekikgW8D9zh7O8LqIgsEJHVInJPfSeLyK0ikioiqbm5uc0cujHmRH1vZDdGJrfnofc22q3LJ6m0ooq84oqAfwYGXEwwIjIJyFHVtOMUuxZ4UVWTgInAyyISgmeOtDHAdc6fl4vI+UefrKqzVTVFVVMSExObvxLGmBPvWJCZAAAXYElEQVQSEiI8PG0QhyqqeeDdDb4OJyBl5h8CAnsW5VputmBGA5NFZCfwGnCeiLxyVJmbgbkAqvo5EAkkAFnAJ6qap6qleFo3w12M1RjTTHolRvGz83vz3tq9vLZyl6/DCTiHZ1G2BNMwVZ2hqkmqmoxnAH+xqk4/qtgu4HwAEemPJ8HkAguAwSLSxhnwHw/Yr0PGBIgfje/FuL6J/Oatdby3dq+vwwkolmBOgYjMFJHJzttfAj8UkTXAq8AN6lEAPAmsAr4CVqvqe96O1RhzcsJDQ/jH9BGM6N6Ou/6bzpJNOb4OKWBk5pcSFRFGuzaBO01/LQmW2wlTUlI0NTXV12EYY+o4WFbJ9/7vCzKyi3npppGc3TPe1yH5vZtfXMWewjI+uHOsV64nImmqmuLGZwf2UzzGGL8WExnOv246i67t23DLS6ls2nfQ1yH5vV35pXRtFxwLBluCMca4qn3bVrxy81mEhwqPfrDJ1+H4NVUNmocswRKMMcYLTouN5JaxPVmyOZc1mQd8HY7f2newjPKqGrrHW4Ixxpgm+8Go7sS2DufpRRm+DsVvZWR7Fm/r3SHax5E0D0swxhiviI4M55YxPVi0KYd1WYW+DscvZeR4EkyfjlE+jqR5WIIxxnjN9aOTiYkM4+nF1oqpz9acItq1CSe+bStfh9IsLMEYY7wmJjKcm8f05KMN2azfY62Yo2VkF9OnYzQi4utQmoUlGGOMV90wOpnoyDAbizmKqpKRU0yfDsHRPQaWYIwxXhbbOpybRvdgwfpsNu6152Jq5RaXU3io0hKMMcacipvG9CA6MownP9ri61D8xtbs2gH+4LiDDCzBGGN8ILZ1OLeO9YzFrN5V4Otw/MLhO8isBWOMMafmpjE9SIhqxWMfbrIlloEt2UXERIaRGB3h61CajSUYY4xPtI0I4/Zze/PF9nyWZ+T5Ohyfy8gJrjvIwBKMMcaHrj2rG13iWvP4gs3U1LTsVszWnGL6BskDlrUswRhjfCYiLJSfX9CXdbsL+eDrfb4Ox2f2F5eTX1IRNFPE1LIEY4zxqcuHdaFPhyj+vHAzVdU1vg7HJ4JxgB8swRhjfCw0RLj7u6ezPa+EeWlZvg7HJ4JtDrJaricYEQkVkXQRmV/PsW4issQ5vlZEJtZzvFhE7nY7TmOM71w4oCNDu8bxl0UZlFVW+zocr9uaXURURBinxUT6OpRm5Y0WzJ3AxgaO/Q6Yq6rDgGuA5446/hTwgYuxGWP8gIhw94Wns7ewjFdX7vJ1OF6XkVNM7w5RQXUHGbicYEQkCbgEmNNAEQVinO1YYE+dcy8DtgPr3YzRGOMfRveO5+ye7fnrkq2UVlT5Ohyv2pIdXHOQ1XK7BTMLuAdoaOTufmC6iGQB7wN3AIhIW+DXwAMux2eM8RO1rZi84gpe+uwbX4fjNQUlFeQVlwfd+Au4mGBEZBKQo6ppxyl2LfCiqiYBE4GXRSQET2J5SlWLG7nGrSKSKiKpubm5zRa7McY3UpLbM+H0RP7+yTYOllX6Ohyv2JobfHOQ1XKzBTMamCwiO4HXgPNE5JWjytwMzAVQ1c+BSCABOAt4zDn3LuA3InL70RdQ1dmqmqKqKYmJia5VxBjjPb+84HQKD1Xyz+U7fB2KV9Quk2xdZCdAVWeoapKqJuMZwF+sqtOPKrYLOB9ARPrjSTC5qjpWVZOdc2cBf1LVZ92K1RjjPwYlxXLRwNP454odFJRU+Doc12XkFNGmVSidY1v7OpRm5/XnYERkpohMdt7+EvihiKwBXgVuUJv1zpgW7xcX9qWkoopHPtgU9A9fbnXuIAsJCa47yADCvHERVV0KLHW276uzfwOerrTjnXu/i6EZY/xQ347R3DS6B/9csYOMnCKeunoo3ePb+josV2RkF3NO73hfh+EKe5LfGOOXfj9pAH+5Zihbc4q5+C/LeXXlrqCb1r/wUCX7DpbRJ8jmIKvllRaMMcacjClDuzCyR3vufn0NM95cx39XZTK6dzwjurdjWNd2tGvbytchnpIvtu8H4IwuMY2UDEyWYIwxfq1TbGtevuksXvnyG15PzeLvn2yn2pna/+ye7XnxxpFEhof6OMqT80ZaFglREYzqGZxdZJZgjDF+LyRE+MGoZH4wKpnSiirWZhXy6dY8nlm8lWcXb+Xu757u6xBPWH5JBUs253D9qGTCQoNztMISjDEmoLRpFcbZPeM5u2c8ew6U8fdPtnHJ4E707xRY3UzvrtlDZbUybUSSr0NxTXCmTWNMi/C7S/oT2zqce99cd7jbLFC8sTqLAZ1iAi4xnghLMMaYgNWubSvuu3QAazIP8K/Pd/o6nCbLyC5ibVZhULdewBKMMSbATR7SmfF9E3l8wWZ2Hzjk63CaZN7qLEJDhClDO/s6FFdZgjHGBDQR4Y+XnYEq/P7tr/3+WZnqGuXt9N1M6JtIQlSEr8NxlSUYY0zA69q+DXd9pw+LN+Wwbnehr8M5rhVb88g+WB703WNgCcYYEySuGdmNVmEhvJGW5etQjuuNtCxiW4dzfv8Ovg7FdZZgjDFBIbZ1OBcM6Mg7a/ZQUeWfE2QWlVWyYP0+Lh3SiYiwwHw49ERYgjHGBI0rhidRUFrJks05vg6lXh+s20d5VQ1Thwd/9xhYgjHGBJGxfRJIiIrw226yt9J30yOhLcO6xvk6FK+wBGOMCRphoSFcNrQzSzbnkO9ni5XtOXCIL3bs57KhXRAJvrVf6mMJxhgTVKaNSKKyWnl3zR5fh3KE/321B1W4bFhwP/tSlyUYY0xQ6d8phgGdYnhjtf90k6kqb6VnMbxbXNAunFYfSzDGmKAzdXgX1mYVkpFd5OtQANiw9yBbsou5vIUM7tdyPcGISKiIpIvI/HqOdRORJc7xtSIy0dl/gYikicg658/z3I7TGBM8pgztQmiIMM9PWjFvp+8mLESYNKiTr0PxKm+0YO4ENjZw7HfAXFUdBlwDPOfszwMuVdVBwPXAy65HaYwJGonREUzom8jb6bt9PstydY3yv6/2MOH0DgG/AueJcjXBiEgScAkwp4EiCtTOVR0L7AFQ1XRVrR2hWw9EikhwT9pjjGlWU4cnkX2wnBVb83wax2fb8sgpKmfq8C4+jcMX3G7BzALuARp6rPZ+YLqIZAHvA3fUU2YakK6q5UcfEJFbRSRVRFJzc3ObKWRjTDD4zoAOxLUJ5/XUTJ/G8Vb6bqIjwjivX/BPDXM01xKMiEwCclQ17TjFrgVeVNUkYCLwsogcjklEBgKPAj+q72RVna2qKaqakpiY2IzRG2MCXURYKJcN7cLC9dkcKPXNMzGlFVUs+HofEwd1IjI8+KeGOZqbLZjRwGQR2Qm8BpwnIq8cVeZmYC6Aqn4ORAIJcLh77S3gB6q6zcU4jTFB6sqUJCqqa3jHR8/EfLQhm5KKai5vgd1j4GKCUdUZqpqkqsl4BvAXq+r0o4rtAs4HEJH+eBJMrojEAe8BM1T1U7diNMYEt4GdYxnQKYa5Puomm5eWRZe41oxMbu+T6/ua15+DEZGZIjLZeftL4IcisgZ4FbhBPasF3Q70Bn4vIl85r5bXgWmMOWVXpSTx9e6DbNhz0KvX3XPgECu25jFtRBIhIS1japijeSXBqOpSVZ3kbN+nqu842xtUdbSqDlHVoaq60Nn/R1Vt6+yrffnn9KjGGL82ZWgXWoWG8Hqad1sxb6XvRhWmtdDuMbAn+Y0xQa5d21ZcMKAjb6fv9to6MarKvLQsRvZo36KmhjmaJRhjTNC7MsWzTsyijdleud7qXQXsyCvhyhawLPLxWIIxxgS9sX0SOS0mkte9tE7MvLQs2rQKZWILmxrmaJZgjDFBLzREmDaiC0s357CvsMzVax2qqGb+mr1cfEYn2kaEuXotf2cJxhjTIlyd0o2wkBDufXMtNS7OT7Zwwz6Kyqu4ooV3j4ElGGNMC9Etvg33XTqApZtz+euSra5d5/XULJLateasHi3z2Ze6LMEYY1qM687qxmVDO/Pkx1tYkdH8k2DuPnCIT7flcUULfvalLkswxpgWQ0T409RB9OkQxZ2vpTfreIyq8vTHGc6zL9Y9BpZgjDEtTJtWYfxt+gjKKqu57T+rm+3ZmKc+2sJ/UzP50fiedG3fplk+M9BZgjHGtDi9EqN49IrBpH1TwMSnl/POmj2ntDDZnOXbeXrxVq5O6cq9F/VrxkgDmyUYY0yLNGlwZ/4+fQQhAj97NZ2LZi3j3ZNINK+nZvLH9zYycdBp/GnqIERs7KWWJRhjTIt10Rmn8eGd43j2e8MAuOPVdKbP+ZLcomPWN6zXwvX7+PUbaxnbJ4Gnrh5KqA3sH8ESjDGmRQsJESYN7syCu8bxyNRBrN5VwKRnlpO6M/+4532VeYCfvZbO4KQ4/j59BBFhLW9BscZYgjHGGDyJ5pqR3Xjrp6OJDA/lmtlfMGf5djwriBwpM7+UW15aRWJ0BHOuT2nxT+w3xBKMMcbUMaBzDO/cPoZz+3Xgj+9t5Pv/XMlXmQcOHy88VMmNL66ioqqGF24YSUJUhA+j9W+WYIwx5iixrcOZ/f0R3H/pANbvKeSyv37KzS+uIn1XAT95JY1v9pfwj++n0LtDlK9D9WtSX/MvEKWkpGhqaqqvwzDGBJni8ipe+mwns5dtp/BQJQB/vnII04JkrjERSVPVFDc+2/WOQxEJBVKB3bWrWtY51g14CYgDQoF7VfV959gM4GagGviZqi5wO1ZjjDlaVEQYt53bm++P6s6/PttJ+7YRQZNc3OaNkak7gY1ATD3HfgfMVdW/icgA4H0g2dm+BhgIdAY+FpG+qlrthXiNMeYYMZHh3H5eH1+HEVBcHYMRkSTgEmBOA0WUbxNPLLDH2Z4CvKaq5aq6A9gKjHQzVmOMMc3L7RbMLOAeILqB4/cDC0XkDqAt8B1nfxfgizrlspx9xhhjAoRrLRgRmQTkqGracYpdC7yoqknAROBlEQkB6nsc9pi7EUTkVhFJFZHU3NzcZonbGGNM83Czi2w0MFlEdgKvAeeJyCtHlbkZmAugqp8DkUACnhZL1zrlkvi2++wwVZ2tqimqmpKYmNj8NTDGGHPSXEswqjpDVZNUNRnPgP1iVZ1+VLFdwPkAItIfT4LJBd4BrhGRCBHpAfQBVroVqzHGmObn9fkNRGQmkKqq7wC/BP5PRH6OpwvsBvU8mLNeROYCG4Aq4Da7g8wYYwKLPWhpjDEtmJsPWtpUMcYYY1wRNC0YEckFvjlqdyxQeIL7GttOAPJOMsz6rn0iZZpSH2/VpbFYGytzonU5+n3tdt199t00LdbGyth349ufAccr50Zd2qqqO3dJqWrQvoDZJ7qvsW0840fNFs+JlGlKfbxVl1Otz4nW5Th1qLvPvhv7bvz6u2lKXZrzu3H731ljr2DvInv3JPY1Zbs54zmRMk2pj7fq0tTPaajMidbl6PfvNlDmZNl3c/z99t1472fA8cr5U10aFTRdZN4iIqnq0oCYtwVTXSC46hNMdYHgqo/VpemCvQXjhtm+DqAZBVNdILjqE0x1geCqj9WliawFY4wxxhXWgjHGGOOKFp1gROR5EckRka9P4twRIrJORLaKyNMiInWO3SEim0VkvYg81rxRNxhPs9dFRO4Xkd0i8pXzmtj8kTcYkyvfjXP8bhFREUlovoiPG48b382DIrLW+V4Wikjn5o+83njcqMvjIrLJqc9bIhLX/JE3GJMb9bnS+b9fIyKuj9WcSh0a+LzrRSTDeV1fZ/9x/1/Vy81b1Pz9BYwDhgNfn8S5K4FReGZ+/gC42Nl/LvAxEOG87xDAdbkfuDtYvhvnWFdgAZ5nphICtS5ATJ0yPwP+HsB1uRAIc7YfBR4N5H9nQH/gdGApkOKvdXDiSz5qX3tgu/NnO2e73fHqe7xXi27BqOoyIL/uPhHpJSIfikiaiCwXkX5HnycinfD8B/9cPX/z/wIucw7/BHhEVcuda+S4WwsPl+riMy7W5yk8axR5bfDRjbqo6sE6Rdvipfq4VJeFqlrlFP0Cz+zpXuFSfTaq6mZvxO9c76Tq0IDvAh+par6qFgAfARed7M+JFp1gGjAbuENVRwB3A8/VU6YLniUFatVdEK0vMFZEvhSRT0TkTFejPb5TrQvA7U7XxfMi0s69UJvklOojIpOB3aq6xu1Am+CUvxsReUhEMoHrgPtcjLUxzfHvrNZNeH479qXmrI+vNKUO9ekCZNZ5X1uvk6qv12dT9mciEgWcA7xep3sxor6i9eyr/Q0yDE/T8mzgTGCuiPR0sr7XNFNd/gY86Lx/EPgznh8AXneq9RGRNsBv8XTH+FQzfTeo6m+B34rIDOB24A/NHGqjmqsuzmf9Fs/s6f9uzhhPRHPWx1eOVwcRuRG409nXG3hfRCqAHap6OQ3X66TqawnmSCHAAVUdWneniIQCtStzvoPnB2/dZnzdBdGygDedhLJSRGrwzPfj7SU3T7kuqppd57z/A+a7GXAjTrU+vYAewBrnP10SsFpERqrqPpdjP1pz/Dur6z/Ae/ggwdBMdXEGkycB53v7l7GjNPd34wv11gFAVV8AXgAQkaV4lkjZWadIFjChzvskPGM1WZxMfd0egPL3F5BMncEx4DPgSmdbgCENnLcKTyuldsBrorP/x8BMZ7svnuamBGhdOtUp83PgtUD+bo4qsxMvDfK79N30qVPmDmBeANflIjxrPyV689+X2//O8NIg/8nWgYYH+Xfg6YVp52y3b0p9643LF1+ov7yAV4G9QCWeDH0znt9yPwTWOP/o72vg3BTga2Ab8CzfPrTaCnjFObYaOC+A6/IysA5Yi+e3tk7eqItb9TmqzE68dxeZG9/NG87+tXjmleoSwHXZiucXsa+cl1fuiHOxPpc7n1UOZAML/LEO1JNgnP03Od/JVuDGxup7vJc9yW+MMcYVdheZMcYYV1iCMcYY4wpLMMYYY1xhCcYYY4wrLMEYY4xxhSUYE9REpNjL15sjIgOa6bOqxTNb8tci8m5jswyLSJyI/LQ5rm1Mc7DblE1QE5FiVY1qxs8L028nZnRV3dhF5CVgi6o+dJzyycB8VT3DG/EZ0xhrwZgWR0QSReQNEVnlvEY7+0eKyGciku78ebqz/wYReV1E3gUWisgEEVkqIvPEs47Jv2vXxnD2pzjbxc6ElGtE5AsR6ejs7+W8XyUiM5vYyvqcbyftjBKRRSKyWjzrc0xxyjwC9HJaPY87ZX/lXGetiDzQjH+NxjTKEoxpif4CPKWqZwLTgDnO/k3AOFUdhmd24j/VOWcUcL2qnue8HwbcBQwAegKj67lOW+ALVR0CLAN+WOf6f3Gu3+h8Ts48WOfjmU0BoAy4XFWH41l/6M9OgrsX2KaqQ1X1VyJyIdAHGAkMBUaIyLjGrmdMc7HJLk1L9B1gQJ2ZZmNEJBqIBV4SkT54ZooNr3POR6pad82NlaqaBSAiX+GZC2rFUdep4NsJQtOAC5ztUXy7lsZ/gCcaiLN1nc9Ow7M2B3jmgvqTkyxq8LRsOtZz/oXOK915H4Un4Sxr4HrGNCtLMKYlCgFGqeqhujtF5Blgiape7oxnLK1zuOSozyivs11N/f+XKvXbQc6GyhzPIVUdKiKxeBLVbcDTeNZ/SQRGqGqliOwEIus5X4CHVfUfJ3hdY5qFdZGZlmghnvVTABCR2mnNY4HdzvYNLl7/CzxdcwDXNFZYVQvxLIt8t4iE44kzx0ku5wLdnaJFQHSdUxcANznrgyAiXUSkQzPVwZhGWYIxwa6NiGTVef0Czw/rFGfgewOeJRYAHgMeFpFPgVAXY7oL+IWIrAQ6AYWNnaCq6Xhmxr0Gz4JcKSKSiqc1s8kpsx/41Lmt+XFVXYinC+5zEVkHzOPIBGSMq+w2ZWO8zFld85CqqohcA1yrqlMaO8+YQGNjMMZ43wjgWefOrwP4aBlqY9xmLRhjjDGusDEYY4wxrrAEY4wxxhWWYIwxxrjCEowxxhhXWIIxxhjjCkswxhhjXPH/jDiXn8khkQMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot(skip_end=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 23:37\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      4.225404    4.029806    0.302210  (23:37)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 1e-02, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('fit_head')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('fit_head');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 8:58:03\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      3.970405    3.867857    0.316788  (26:53)\n",
      "2      3.920167    3.786381    0.327380  (26:54)\n",
      "3      3.851934    3.726164    0.336203  (26:53)\n",
      "4      3.814150    3.671940    0.343449  (26:54)\n",
      "5      3.798842    3.626528    0.348548  (26:53)\n",
      "6      3.753443    3.585267    0.353221  (26:57)\n",
      "7      3.723497    3.546739    0.357538  (26:55)\n",
      "8      3.676399    3.515404    0.361083  (26:53)\n",
      "9      3.668247    3.487297    0.364337  (26:52)\n",
      "10     3.633528    3.454587    0.368336  (26:51)\n",
      "11     3.611737    3.425386    0.372090  (26:49)\n",
      "12     3.586109    3.396924    0.375549  (26:54)\n",
      "13     3.568666    3.371741    0.379174  (26:53)\n",
      "14     3.536366    3.337895    0.383549  (26:59)\n",
      "15     3.507154    3.313276    0.387211  (26:56)\n",
      "16     3.465269    3.290863    0.390346  (26:56)\n",
      "17     3.479237    3.274435    0.392694  (26:54)\n",
      "18     3.407071    3.261680    0.394350  (26:55)\n",
      "19     3.425704    3.255773    0.395224  (26:52)\n",
      "20     3.390900    3.253836    0.395450  (26:52)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(20, 1e-3, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(\"afteroom\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load(\"afteroom\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3.2538004, tensor(0.3954)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.validate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save_encoder(\"enc_best\")\n",
    "learn.save(\"lm_best\", with_opt=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEKCAYAAAA1qaOTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xd4VNXWwOHfIhVI6KGjoXcIISII0kUBRVRUUGyoXFGvBb1X7Mpnwd6uvYAVFBtKVZogIBB671E6obdA2v7+OCdhkswkM5OZzCRZ7/PMw+TMKSuTkDX77L3XFmMMSimlVGGVCXQASimlSgZNKEoppXxCE4pSSimf0ISilFLKJzShKKWU8glNKEoppXxCE4pSSimf0ISilFLKJzShKKWU8onQQAfgqWrVqpnY2NhAh6GUUsXKsmXLDhpjYvx5jWKXUGJjY0lMTAx0GEopVayIyN/+vobe8lJKKeUTmlCUUkr5hCYUpZRSPlHs+lCUUiVHWloau3bt4syZM4EOpcSIjIykbt26hIWFFfm1NaEopQJm165dREdHExsbi4gEOpxizxjDoUOH2LVrF/Xr1y/y6+stL6VUwJw5c4aqVatqMvEREaFq1aoBa/FpQlFKBZQmE98K5PtZahLK0qTDvP7bJlLTMwMdilJKlUilJqEs//sIb8/eSnqmJhSlFBw6dIi4uDji4uKoWbMmderUyf46NTXVrXPcdtttbNq0yc+RFh/aKa+UKpWqVq3KypUrAXjmmWeIiori4YcfzrGPMQZjDGXKOP/sPXbsWL/HWZyUmhaKUkq5Y+vWrbRq1Yq77rqL+Ph49u7dy/Dhw0lISKBly5aMHj06e98uXbqwcuVK0tPTqVSpEqNGjaJt27Z06tSJAwcOBPC7CIxS10IxJtARKKWcefbXdazfc9yn52xRuwJPX9HS4+PWr1/P2LFj+eCDDwAYM2YMVapUIT09nR49ejBo0CBatGiR45hjx47RrVs3xowZw8iRI/nss88YNWqUT76P4qLUtFB0IIlSyl0NGzbkggsuyP56/PjxxMfHEx8fz4YNG1i/fn2eY8qWLUvfvn0BaN++PUlJSUUVbtAodS0UpVRw8qYl4S/ly5fPfr5lyxbeeustlixZQqVKlRg6dKjTeR7h4eHZz0NCQkhPTy+SWINJqWmhKKWUN44fP050dDQVKlRg7969zJgxI9AhBa1S10LRLhSllCfi4+Np0aIFrVq1okGDBnTu3DnQIQUtMcWslzohIcF4s8DWx/O28/zUDax99lKiIkpdHlUqKG3YsIHmzZsHOowSx9n7KiLLjDEJ/ryu3vJSSinlE5pQlFJK+USpSyjF7RafUkoVF6Umoeg8FKWU8q9Sk1CUUkr5lyYUpZRSPlHqEor2oCilsnTv3j3PRMU333yTu+++2+UxUVFRAOzZs4dBgwa5PG9B0xvefPNNTp8+nf11v379OHr0qLuhB6VSl1CUUirLkCFDmDBhQo5tEyZMYMiQIQUeW7t2bb7//nuvr507oUydOpVKlSp5fb5goAlFKVVqDRo0iMmTJ3P27FkAkpKS2LNnD3FxcfTq1Yv4+Hhat27NpEmT8hyblJREq1atAEhJSWHw4MG0adOG66+/npSUlOz9RowYkV36/umnnwbg7bffZs+ePfTo0YMePXoAEBsby8GDBwF4/fXXadWqFa1ateLNN9/Mvl7z5s258847admyJX369MlxnWCgU8aVUsFh2ijYt8a356zZGvqOcfly1apV6dChA9OnT+fKK69kwoQJXH/99ZQtW5affvqJChUqcPDgQTp27MiAAQNcrtf+/vvvU65cOVavXs3q1auJj4/Pfu3555+nSpUqZGRk0KtXL1avXs19993H66+/zpw5c6hWrVqOcy1btoyxY8eyePFijDFceOGFdOvWjcqVK7NlyxbGjx/Pxx9/zHXXXccPP/zA0KFDffNe+YC2UJRSpZrjba+s213GGB577DHatGlD79692b17N/v373d5jnnz5mX/YW/Tpg1t2rTJfu27774jPj6edu3asW7dOqel7x39+eefXHXVVZQvX56oqCiuvvpq5s+fD0D9+vWJi4sDgrNEvt9bKCISAiQCu40xl+d6bSRwB5AOJAPDjDF/+zMendeoVJDKpyXhTwMHDmTkyJEsX76clJQU4uPjGTduHMnJySxbtoywsDBiY2Odlqx35Kz1smPHDl599VWWLl1K5cqVufXWWws8T36TryMiIrKfh4SEBN0tr6JoodwPbHDx2gogwRjTBvgeeNlfQbhqqiqlSreoqCi6d+/OsGHDsjvjjx07RvXq1QkLC2POnDn8/Xf+n3O7du3K119/DcDatWtZvXo1YJW+L1++PBUrVmT//v1MmzYt+5jo6GhOnDjh9Fw///wzp0+f5tSpU/z0009cfPHFvvp2/cqvCUVE6gL9gU+cvW6MmWOMyRrm8BdQ15/xKKWUM0OGDGHVqlUMHjwYgBtvvJHExEQSEhL4+uuvadasWb7HjxgxgpMnT9KmTRtefvllOnToAEDbtm1p164dLVu2ZNiwYTlK3w8fPpy+fftmd8pniY+P59Zbb6VDhw5ceOGF3HHHHbRr187H37F/+LV8vYh8D7wIRAMP577llWvf/wH7jDHP5XdOb8vXf/rnDv5v8npWPd2HimXDPD5eKeV7Wr7eP0pc+XoRuRw4YIxZ5sa+Q4EE4BUXrw8XkUQRSUxOTi5cYNqHopRSfuHPW16dgQEikgRMAHqKyFe5dxKR3sDjwABjzFlnJzLGfGSMSTDGJMTExHgVjPagKKWUf/ktoRhjHjXG1DXGxAKDgdnGmBwDpkWkHfAhVjI54K9YlFLBS5eU8K1Avp9FPg9FREaLyAD7y1eAKGCiiKwUkV+KOh6lVOBERkZy6NAhTSo+Yozh0KFDREZGBuT6RTJT3hgzF5hrP3/KYXvvorh+jli0E0WpoFG3bl127dpFoftGVbbIyEjq1g3MgNlSU3pFp6EoFXzCwsKoX79+oMNQPqKlV5RSSvlEqUko+49bA8hOnEkPcCRKKVUylZqE8sEf2wD4ZdWeAEeilFIlU6lJKFkyM7VTXiml/KH0JRTNJ0op5RelLqG8MXMza3cfC3QYSilV4pS6hAJw3YeLAh2CUkqVOKUyoZxOzQh0CEopVeKUyoSilFLK9zShKKWU8glNKEoppXxCE4pSSimfKLUJ5ctFSYEOQSmlSpRSm1CenLSOn1fsDnQYSilVYpSahFKvStk82x74dmUAIlFKqZKp1CSUX+/t4nS7MYZF2w6x79iZIo5IKaVKllKTUCqVC3e6/ff1+xny8V/0fWteEUeklFIlS6lJKK4M/3IZAEdOpwU4EqWUKt5KfUJxlJqeGegQlFKq2CpVCeWP/3TP9/UmT0xj64ETRROMUkqVMKUqoZxftXyB+zw0cTUZumiKUkp5rFQlFICGMfknlVU7j/Lt0p1FFI1SSpUcpS6h/HJvF6IiQvPd57Gf1jB97T5W7jxaRFEppVTxV+oSSvmIUBKf6F3gfnd9tYyB7y4ogoiUUqpkKHUJBSAyLCTQISilVIlTKhMKwFOXt3Brvx+X7+LAcZ1Fr5RSBRFjiteIpoSEBJOYmOiTc8WOmuLR/pueu4yIUG3dKKWKHxFZZoxJ8Oc1/N5CEZEQEVkhIpOdvBYhIt+KyFYRWSwisf6OpzAOnkwNdAhKKRW0iuKW1/3ABhev3Q4cMcY0At4AXiqCeLLd3Ol8j/Y/ckoTilJKueLXhCIidYH+wCcudrkS+Nx+/j3QS0TEnzE5erx/86K6lFJKlXj+bqG8CfwXcFUkqw6wE8AYkw4cA6rm3klEhotIoogkJicn+yy4iNAQZj/UjY4Nqri1/4ivl7H3WIrPrq+UUiWJ3xKKiFwOHDDGLMtvNyfb8owSMMZ8ZIxJMMYkxMTE+CxGgAYxUbwzJN6tfXceTqHvW/N9en2llCop/NlC6QwMEJEkYALQU0S+yrXPLqAegIiEAhWBw36MyamsmfNt61UqcN+jp9Po99Z81u05xvwtvmstKaVUcVckw4ZFpDvwsDHm8lzb7wFaG2PuEpHBwNXGmOvyO5cvhw07OnIqlejIUBo9Ps2j45LG9Pd5LEop5WslYthwbiIyWkQG2F9+ClQVka3ASGBUUceTpXL5cEJDynicII6l5FyYK/nEWR7/aY2uraKUKnXyr5LoI8aYucBc+/lTDtvPANcWRQz+8uC3KwktI9xyUSydGlTlgudnAtCxQVWuaFs7wNEppVTRKZKEUtzc0aU+n/y5w619Z288AMBv6/czpEO97O0G+PvQKdIyMmlUPdofYSqlVFAptbW88vPE5S1YP/pSBrWvy4c3tXf7uPFLcq6j0u2VufR+fZ6vw1NKqaCkCcWFcuGhvHptWy5tWZPZD3Xz+Pi5dstFKaVKC00obggL8fxt+nHFbj9EopRSwUsTihtioiMKdfwPy3b5KBKllApemlDcUNgFuR6auIqMzOK1TIBSSnlKE0oRufeb5YEOQSml/EoTioeqe3n7a9rafU63p2Vk6iRIpVSJoAnFTTdceB6NqkexcFRPn5638ePTaPLENNbtOebT8yqlVFHThOKmF65qzcyR3QgNKeNWEUlnXv9tE3M3WcOJ9x8/w8B3F2S/1v/tP30Sp1JKBYrOlPfCpHs6A56vSf/27K0AXNSwKgu3HXK6z/EzaYSHlCn0QACllCpq2kIJAFfJ5NTZdNo88xvNnpxOSmoG25NPFnFkSinlPU0ohfDqtW2znzeuHlXo87V8ekb28+ZPTafna3/w0vSNZOqQY6VUMaAJpRAGtK1Nv9Y1mTmyG1fF18neflvnWJ9d4/252xj80V8+O59SSvmL9qEUQnhoGd670Soe2aBaQwa0rU3dyuUASE3P5OvF//jkOkuSinwRS6WU8pi2UHykTBnJTiYAz1/Vmk4NqhZ5HAeOn2HdnmNkZhpemLqBvcdSijwGpVTppC0UP6pbuaxfz5+eYU2IDLWLVxpj6PDCLAC+v6sTH83bzupdR5kwvJNf41BKKdAWivtSjnp8SKVyYX4IBBZuPUhmpqHts7/R8cXZACQdPMXoyeuz98nqx0/P0A59pVTR0ITijr8XwlttYPNvHh32UJ+mPDewlU9DmbPpADd8sphXf9vEqdQMDp48C0D3V+cydkFS9n5ZrZdTqRk+vb5SSrmiCcUdMc2gcix8eyNsmu72YZFhIQzteL5PQ0k+YSWQ9+Zuy942bNzSPPvd8MliADbsPe7T6yullCuaUNxRrgrcPAlqtIRvh8JGz2bI+1JoGcmzbbauDqmUCgKaUNxVtjLc9DPUagPf3Qzrf3H70Gvi6+bZ9v1dnnWUr9tzjHV7jrF2t7Y4lFLBSROKJ8pWgpt+gtrxMPFWWPeTW4c9f1Urnrq8RY5tCbFVsp93bxpT4Dn6v/0n/d/+k88W7PAoZKWUKiqaUDwVWRFu+hHqXgDf3w5rvi/4kLAQhnWpz5VxtZ2+fl6Vck63Z2lRq4JXoSqlVFFyK6GISEMRibCfdxeR+0TEuxruJUFENAz9Ac7rCD/eCau/c+uwZwe0BKBBTHkA7uvZCICbO8UCMPa2C+jVrDpT7uvCX4/2yj5uvXasK6WKATGm4HkKIrISSABigRnAL0BTY0w/v0bnREJCgklMTCzqyzqXegq+uR7+XgBXvgdxQwo85ODJs5QLD6FceMFzSj0tj+9K0pj+Be6z8/Bp6lYui4iQkprBiTNpVK8Q6ZPrK6UCT0SWGWMS/HkNd2fKZxpj0kXkKuBNY8w7IrLCn4EVC+Hl4YbvYPxg+HkEmAxoNzTfQ6pFub+EcNKY/j5LKs70fWt+jmHFj/Vrxvtzt3HkdFr2tu0v9KOMk5FlSimVm7t9KGkiMgS4BZhsb/PPNPDiJrwc3PAtNOwBk+6FZZ8HOqJ8fb4wiW+XWkUrc89ReWHqxhzJBGD/iTNFFptSqnhzN6HcBnQCnjfG7BCR+sBX+R0gIpEiskREVonIOhF51sk+54nIHBFZISKrRaTIb6H5RFhZGDweGvWCX++DxM8CHZFTG/Ye5+lf1vHID2sY+d1Kt475cfluP0ellCop3Eooxpj1xpj7jDHjRaQyEG2MGVPAYWeBnsaYtkAccJmIdMy1zxPAd8aYdsBg4D0P4w8eYZEw+BtofClMfhCWfBzoiPLo+9b87OfuJoo0u4SLJ06eTff4GKVU8efuKK+5IlJBRKoAq4CxIvJ6fscYS9YatmH2I/cIAANkjYmtCOxxO/JgFBoB138JTfvB1Ifhrw8CHREAp1PT2bjPu5FiniaU+VuSafX0DBZuO+jV9ZRSxZe7t7wqGmOOA1cDY40x7YHeBR0kIiH2CLEDwO/GmMW5dnkGGCoiu4CpwL9dnGe4iCSKSGJycrKbIQdIaARc+zk0uxymPwKL3i2yS5cPD3G6/fZxiVz25nynrxXE02rFi7YdAuCDP7Z7dT2lVPHlbkIJFZFawHWc65QvkDEmwxgTB9QFOohI7tK7Q4Bxxpi6QD/gSxHJE5Mx5iNjTIIxJiEmpuBZ5QEXGg7XjoMWV8KMx2DB2z6/RI9cs+uTxvRn3n97MOW+LvRpUSPHa4u2H/L6OqkFtFCSDp7iwIkzfPDHNmJHTckuWjlvczJ/btFWilKlibsJZTTW/JNtxpilItIA2OLuRYwxR4G5wGW5Xrod+M7eZxEQCVRz97xBLSQMrvkMWl4Nvz8JMx6H9FSfnX7sbR2yn3/3L6suWNWoCFrWrshHNye4nJXvqaiIvCPLD548S2q6lWi6vzqXDs/PYsy0jXn2S/xbly5WqjRxax6KMWYiMNHh6+3ANfkdIyIxQJox5qiIlMW6RfZSrt3+AXoB40SkOVZCCfJ7Wh4ICYWrP7aqFS/6n7WuyqBPoUoDr0639fm+GCBEcs4LqVI+7wjutwa3Y9LKwndJZSUORwnPzXTrWEHnryhVmrjbKV9XRH4SkQMisl9EfhCRvCV0c6oFzBGR1cBSrD6UySIyWkQG2Ps8BNwpIquA8cCtxp2p+8VJSCj0fw2u+xIOb4MPurpdqsXRumcvJTSkDGEhZbInGi59vDfPDWxFo+rRvo4625k07xfoWpLk/a02pVTx4+5M+bHAN8C19tdD7W2XuDrAGLMaaOdk+1MOz9cDnd0NtlhrMQBqt7Nqf/14J2ybA/1egYgotw4v7+TWU0x0hM8X8MrtbK4Wiif5fsFWTShKlSbu9qHEGGPGGmPS7cc4oBj0jgeZSvXglsnQ7RFYPQE+7Ap73JtgGCi5E8qD37ofb00Pa4GlZ2Ry3/gVusqkUsWUuwnloIgMtYcBh4jIUEA/fnojJBR6PAa3/AppKfBJb1j0Hrj45P/pLQlOF+gqKmfTM8jMNNz99TJ+XbWHnz3ol7myneuBAWfTM3hq0lqOnrYGKrw1cwv3f7uSX1btyTEBUylVfLibUIZhDRneB+wFBmGVY1Heiu0CIxZA40tgxqPwzXVwKu8w217Na/DadW2LJKS3BscBMLzruUEDZ9MyefbXdUxds49/j/esHmibOq5XOJi0cg9fLPqbl6ZvAuCNmZuZsnpv9utJB095dC2lVOC5W3rlH2PMAGNMjDGmujFmINYkR1UY5apY5Vr6vQrb/4D3L4Ltc4s8jMQnerPsid5cGVeHpDH9eaxf8+zXlv9zhM8X/e3VeXcfPZ39fM7GA6Tbc1pmbdjP+j3Wba30jExemLohz7HT1u4r8PwpqRl8Mn87GZmGy9+Zzx2fB8myBkqVUoVZsXGkz6IozUSgw51w5yxrNcgvBsLMZyEjreBjfWDxY72oFhVBVRdl9XNXH/ZE1pzIP7cc5LZxS3l7ljV16fbPExm3MAmAict28dG8vLPqX5qed15Lbq/+tonnpmxgypq9rN19nJkb9nsdq1Kq8AqTUHSSgS/VbA3D50L8TfDn6zC2LxxJ8sulhnSoB1iz7Wu46Djv1qTwYy5ioq0klXzSKoE/f6tvZ84fPHkWgIxMzwtYKqV8rzAJpWTNFwkG4eVhwDsw6DNI3gQfXAxrf/D5ZZrVtOpxXpdQz3UooYX51bBk2gMNth2w+kNW/HPUpwuGnU2zEklE6LkaZpmZ+mupVKDk+1dDRE6IyHEnjxOAb2p7qLxaXQN3zYeYpvD9MPhpBKQc8fg0dSqVdbp9477jJI3pT9/WtVweu/PwaZevuStrzsoBPy3StWn/CQBW7Tyave2vHZ4PPvxiUZJbt9iUUvnLN6EYY6KNMRWcPKKNMe5OilTeqBwLt02Dix+G1d/Cux1h41SPThERdu7H+9PdF2U/79mshrPdcziW4n7fSes6FZ1uzzRW/8l3ibvcPpcndtgjwT506INZ8c9RjnnY7/PUpHW8bxe1VEp5r/D3NZT/hIRBryfhztlQvhpMGALf3+50eLEzoQ5rwbc7rzI/jOhE5XJhdIitUuCxvZpXdzvMIR3OY9I9nWlc3Zr1/8b11jBnY2Dop7lXLCic39fvZ/Sv612+/sqMTVz57p9uncsYwzuz3K5xqpQqgLYyioPacXDnHPjzDZj3Cmy3y7a0vNoaJeZCtagINu8/yb09GgHQ/vwqrHiqj1uXjAx1vraKM1WjwmlbrxK//rsL09fu46KGVQHfd5anZWRy5xfW0ODPFuxwuV/SIfdu1z3+81q+WfyPT2JTSmkLpfgIDYfuj8C//oBK51l9K98OhROu52uUsZNNh/oFt0hyq1QubwVjZ766/cLs9Vciw0IY2K5OdvHKVA8X58rPiTNpNH58WqHOcTo1ndhRU5i6Zi8z1+/XZKKUj2lCKW5qtITbZ0LvZ2HL7/DuhbByvNPSLXdcXB+AVi76OPJzlRvlXq5LqEuXxtWQXK2krEQ2d9MBj6/ryrBxSwt9jp2HUwB44/fN3PFF3kmQ3y3dyZFTvluzRqnSRm95FUchodDlAWjWHybdAz/fZQ0vvuJNqHguEXRvWp2kMf29ukQtF/NTxt56AZ0aVmXR9kN0bex8rkpW1818H6zYuPdYCrM3HmBpkuej3HIz9kj3TBd10/77w2o6rqjChOGdCjzXlv0nqF+tPKEh+plMqSyaUIqzao2tkWBLPoZZz1ojwfr8H7S/Nd++FXeUKZP3eMfk1KOp6057Xy6s1enF2T47V5Ztya7rhCUdLLj/ZXvySS55Yx7/6taAyuXCGTNtI2NvvYDuTWPytNaUKk3041VxVyYEOt4FIxZanfeTH4AvBsBh153W3rjhwvPc3le8+K1aOKonAE/0b17Ant6bsGRngfvsO17wnJndR61bZ2t3H+OVGVZxy9vGLfXJCplKFWeaUEqKKvWtkviXvwm7V1iFJv/6AHw00urpK1q4vW8ZLz6lVygbxrYX+nF7l/rZ256atNbj8zhz8mw6M9bty64fVhjGGMYusM6TmZmz/pCu46JKO00oJYkIJNwG9/wF53eG6Y/AZ31g55JCnbZvq5o5ypsUxMndMpd+GNGJq+PrUD48hJAykuOW0RdeVjnOrdXTM/jXl8t8cq6B7y5g9kZrsMGi7YdIdyj18qGTIpdKlSaaUEqiinXhxokw8AM48jd8egl8exMc8mw2eK9m1RnWuT7vD23v0XH5tVCevqIFSWP6c0eX+nx8cwLtz6/C69fFBW3fw+hf12fXHzt+Jo1Vu4759PwLtx7UtV9UiaGd8iWVCMQNgeZXwKL/wYK3YdNUSLjdWoK4fNUCT/HprRd4fWlXbuts3dJ64nL3b6G54+VBbViy4zDfL7PKvPR8bS4P92nKnI2FG7rsOIGyzTO/FepcuRljuOETq5LAwlE9qe2i9ppSxYW2UEq6iCjoPgruWw7thsLSj+HtOJj/mrUEsR/4cpSXO5rUiOK6hHq8em1b+rWuCcD25FPc/fVyJi7zvI6YcTKs+LnJrsu9FHQsWLP8z6Rl5Nh28mx69vOLxvh+NJtSRU0TSmkRXROueAvutvtXZo2Gd9rDym8gM6Pg4z3grIXi7sx7Tz3WrxmfObSkQsoU/lfaWU745E/3Rs3NczH3ZsD/FtDsyekATF69h9hRU2jt4xaPUoGmCaW0iWkKN0yAW6dAVA34eQR82A22zvLZJUKd9Mr/8XAPrydZ5vbCVa0BqFo+nOFdG1K3crl8r+2pwhSMSU13PqrOcQTYvd+scLrPIXvBMKWKK00opVVsF7hjFlzzKZw9Dl9dDV9eBfvWFPrUzjrYK/qwhXLDhefx+4Nd+X1ktzyveTNkObdMYxj1w2q+WJRU6HPl5uqWGMD+4+4nlMOnUhn0/kL2HfPPWjNKeUMTSmlWpgy0HgT3LoVLX4Ddy61VIn8aAcd2++wyT/qoAz4qIpSZI7sC0LhGNFXKh+fZx5sWSvx5lXJ8bQxMWLqTpyat8/hcjpdPSc0gdtSUHEUoH/h2pctjP/dgnsxzk9eT+PcROr44i9vGFm5YuFK+oglFQWgEdLoH7l8JF90La7+Hd+Jh5jNw+rBXpwx3qHHlOFmxMKbdfzGNqkfnu09GPi2ALIsf68WMB7pmfz321g45XjeFuOm1JOnc+3XPN8sBeOync62+/GbTf5uY/0z+7cknSc+wbqn9uOJcwp+zKdmrWJXyNU0o6pyylaHPc3BvIjQfYK2/8mYbmP2cx4llzn+6+zw8d+5mfe/GqK4aFSJpWjOa6Q9czIO9m1CxXBi9m9egWxOr2KUbOcmlrGrFd3yemD0B0hd2Hj5Nz9f+oNHj09iefNJn51XKl/yWUEQkUkSWiMgqEVknIs+62O86EVlv7/ONv+JRHqh8PlzzsVUfrFEva1EvDxNL2TD3Z9bnZ0Db2nxycwKj+jajjo/naTSrWYH7ezcG4JNbEriwgbVuzJIdniXPns3OFcq8vE1tAGZu2O+jKOGGj//i4pfnnLvea3/k2Scz03drzyjlLX+2UM4CPY0xbYE44DIR6ei4g4g0Bh4FOhtjWgIP+DEe5akaLeG6z/Mmlln/V2BiKRduJZRr2xe8rkp+3hocR+8WNbirW0OfzKYffEE9l6+F2Oe/+TPP+iT+d0O7QsVUkIXbDhW4z/EzaV6de/fRFA7rGjDKR/yWUIwlq20eZj9yf4y6E3jXGHPEPsZ39wiU72S5UZseAAAgAElEQVQnlkVWYpn/aoGJJTIshDXP9GHMNW0KdWlfl2TJb4CAt2ublAsP5Y3r2wLu9eF44pdV7lUwDvMy9s5jZtPh+ZleHatUbn7tQxGREBFZCRwAfjfGLM61SxOgiYgsEJG/ROQyf8ajCqlGi3OJpXFva7Z9PoklOjKMEC9GXQ3v2gCAUX2bFTrkLElj+pM0pj/lI1xXG3JnhNjaZy/l8X55S+w3irEGC2RkGJ/MJ9m47zixo6Zw33jnc1ZyW7XzqEfnd7xFlq63y5SP+DWhGGMyjDFxQF2gg4i0yrVLKNAY6A4MAT4RkUq59kFEhotIoogkJifriJaAq9ECrh1n3QrLTiytrdn3Xo4Kc/RYv+YkjenPXd0aFj5WDxTUd3J5m1pERYRyZ9cGLLDXb8mSlTinrtlL++c8+8R/ftVyebZd9uZ8j85xm4slkj+et50/c83en752Lw0em5pd9FIpXymSUV7GmKPAXCB3C2QXMMkYk2aM2QFswkowuY//yBiTYIxJiIlxvuysCoAcieUSmP+6TxNLUftru+u+ig6xVXhnyLm+ktyDDrISiuNwXnd9fluHgncqwFkXM/Sfn7qBoZ8uJjPTsP/4GY6dTuOur5YX+npKOeO3asMiEgOkGWOOikhZoDfwUq7dfsZqmYwTkWpYt8B0UYniJiuxdF0P8162EsviD621WTreAxVqFVkoZcNCSEnzrjbZIRed07Me6kb9quVz9OeEh+b8LObNrb0s5SJ8MyIuP+//sS17dcnCMsYE7XIDKrD82UKpBcwRkdXAUqw+lMkiMlpEBtj7zAAOich6YA7wH2NMwUNaVHDKSix3L4Iml8Kid+GtNjDpXji4pUhCcEwmV8bVZtsL/dj2Qj+vzxcdGUrDmCjK5EoYYSHW11F2n4y3NcT+3bNRjsXLtiWfdGsujad8lUwmrdxN/UensvPwaZ+cT5UsfmuhGGNWA3nGUxpjnnJ4boCR9kOVFNWbw6DPoOeT1losK76yHs0vh84PQl3PFuzyxle3X0jnRlUL/Ul6usOMekcRoSE83KcJl7SwyuW700IZ0qEe4+117Wc/1I335m7j/l6Nc3SK93IyxySY3D/BKh2z5cAJ6lXJ2/ejSjedKa/8p0p96P8aPLAWLn4IdsyDT3rCuMth68zCTUl3oZ1dl6tu5bIeJ5O6lXNOnLytc2y+kynv7dmYpjXzLwXjyLEETYOYKF69ti2hIWW8HvKb26GTZ5mz6QD/mbiKPm/8kV2mxR98sUyAKnl0xUblf1Ex0OtJ6PIALBsHi96Dr66Bmq2h8wPQYiCE+OZX8YOh7Zm6Zi+x1cp7fOyTl7fIsfb8Y06GB7uSX24Ugd8f7Eaj6lH8q2sDKpTNWXnZ2+4XEXjksmaMmbYRgJs+XcJ6hzL5SYd8d1vq0MmzvDN7a/bX787eSp1KkQXWVlOli37MUEUnIhou+jfcvwqufBfSzsAPt1uFKJd87JMVJGtUiMxeZthT1aIisp+fX7WcRy2HkBDnWaFD/SrseLE/japHAfBov+bc06NRjn28vS3XpVG1HK2erQdy1vjq/brnt89+WbWHcQt2MHtjztIxd3+9nHEO1ZCXJB2m9+vzmLJ6r8fXUCWXtlBU0QsNt5YjbnuDtc79n2/A1Idh7hjoeBdccIdVqLKIVXRoOYy8pIlHx4Y5SSgT7+rEBbFVCh2XKx/dlJAj6aX64BaX40TKJ/o3p1uTGC55Y57L/e/5ZjmXtuzrdpWBI6dSOZ2W4fO6bCo4aAtFBU6ZMlZH/R0zrRUka7ezClC+0QqmPGStz+KHfhZXsloRYI3u8kT16Mg82/yZTADKhvt2uPGRXMOmn5uygT82FzyR2NVM+w17j7PnaM5W50VjZtN5zGyOnk5l2d9HvA9WBSVtoajAE7FWkIztYq0YufAda1TY0k+geguIuxHaXG/1xRSR9uf5Nxl4qnp0BAdO+HeJ4M4vzc6z7bkpGwo8LtNF0u/7ljXbP2vp59T0zOxh3Z3HzOZUagY7Xuync1pKEG2hqOBSszVc/RE8tAn6vw5hZeG3x+H1ZjD+Btg4BTK8q6zrjg9vak+LWhU8bqH4w38ubZr9/PcHuzFzZDdmPNA1zwqTvnI61bsJoRlu1AIb/et6mjwxLfvrU/a1irABqopA4P/XKOVM2Upwwe3W48AGWPk1rPoWNk2B8jFWiyXuRmsypQ9d2rIml7asWahzzH6oGw1iogresQD39GjEoPZ1KRceQnRkGBXLWX08E++6yGWrwF3/6taAD/84V5Tijs8TvT5XQQklv5phmcZQBm2hlBTaQlHBr3pzayXJkethyASodyEs/gDe7wQf9bBujaUEz/14XySTLDUqRBIdmXOYcUgZKdTclRHdG/JAryZMGN6RtvWs1k5hFgTL3YdyOjWdT+a7V0HJ3UrHf2xO5kxaBolJh3lnVtFUXVCe0xaKKj5CwqBpX+txMhnWfAcrvrY68Kc/ZnXwx90IDbpDGf/Xx8otq68gWO14sR/GkF1GpmODqlzUsKrHpe9zy91CeXjiKqau2efWse4klA17j3PLZ0sYfEE9Jiy1Kg38u1eeGrIqCGgLRRVPUTHQ6R4YsQCGz4X4m2HrLPjqanijJUwbBTuXFtub9G8P8f0qkCKSpyZZiA86xI+cTuXk2XRiR01hW/JJt5MJwFk3CnlmrSiZlUxU8NIWiireRKzhxrXbWbfFNk2FNd9D4qew+H2oeB60HAitroZacdb+QSgqIpSTZ9MBaBhTngFta/vs3CO6N6SZixIxh04VfuTYkh2Hs9dc8bQW2VuztjD6ytzLJMHnC5M4fCqValHhPDlpXaFjVEVDWyiq5AiLtBLHkG/gP1th4PsQ0xT+eg8+6g5vt7PWatm3NuhaLgse6Un/1laZ/8KuoBgdGcqch7vTIMYqP/PIZc24Mq6O03037TtRqGsBbNx3wuuhv2t3H8uz7ejpVJ7+ZR1vzdriMpk8PHGV29fYcfAUv6/3vo9IuU9MkP3HKkhCQoJJTPR+RIoqhU4fhg2/wrofrQKVJhOqNYGWV1sJKKZpwecoAmfSMmj25HTeGhznMgE4k3sU1YonL6Fy+XAOnjzLpn0n6NyomtvHuuO1a9vykAd/0AuSNKY/q3YepXGNKMqGhVD/0aluHffDiItof37BFRWyvsfSPudFRJYZYxL8eQ1toaiSr1wVaH8L3DwJHtpsVUAuXx3+eAne7QDvd4Z5r8LhwK7tFhkWQtKY/h4lk9wuaVGDyuXDAas2WX7JxFvnVy1H0pj+OVawLIzjZ9K48t0F/PubFS5XnnRm4VbrNtvBk2c5etr54miOlv9TuMEHqmCaUFTpEhVj1Qq7bQqM3ACXjYGwcjD7/6xbYh/1gMTP4Mzxgs8VhN6/Md4v5136eG/evSEeEWhi98cUdi5MlqveXQDArI0HPLqVNX7JPwAkPDeTuNG/F7j/dR8uYvSv61m47SDHz/hvcmxppp3yqvSqUAs6jrAeR3fCup9g1QSY/CDMeMK6Hdb+VqjTPmg783Nzt0ijJ269KJaY6Aj6t6lF/zbnhkb7KqFsSz6V/XyyB9WL9xw749F1MjINny3YwWcLdgDBP8y7ONKEohRApXrQ+T6rvP7uZbBsLKz9AVZ8CTVaQfwt0OY6awZ/KXJfr8YuKy/vOuz5cgO1Kkay18NE4I3MTMPptAyed6MWGVi33Y6dTtNVKAtJE4pSjkSgboL1uPRFWPs9LPscpv0Hfn8SWl5lJZfzOgZdq8WbhbreHtIuR8l6Rx3qV8m3jP/Xi//x+HqLHu3F0qTDJB08xX++X+3x8e5q8Jh7HftZ+r01n11HUpy2WpIOnqJ2pbKEh2oPQUE0oSjlSmQFSBhmPfashOWfw+qJsGo8VGtqdfS3HWJ1+geBKfdd7PExA9rWzpNQWtWpwIhujejfpla+x1YpH86+4+63Nr68vQNglfVvVbuizxPKnV8kEhMdQZoHHftZdh2xWlurdx0lPdNQr3I5pq/dS/KJs7xtr1Q566FuNHSjrE5GpmHguwu4v1djereo4XEsxZkmFKXcUTvOevR5Dtb+aCWXGY/BzGeg+QArucReHJBWy1+P9uJYSppH69u7Mv7OjnRqWNWtfR2XG87P7w92pXGNnLGFeLvucS4pDhWSfTHXZMD/Frh87ZP5O3jx6tYFnuPkmXTW7D7GHV8klrp+Gk0oSnkivDzE32Q99q+D5V9YLZa130OVBlYV5NbXQtWGRRZSzYqR1KyYd4EvT3n6x+9fXRvw4TznQ627N41h7qZkWtSqkCeZAIT6KKE0f2q6T87jjvFL/mHL/hN8c2fHfG9/icNLN326mAtiq7D/+Bn6t6nFRQ2rsXb3MWpWjMyx5HRJoQlFKW/VaAl9X4Lez8D6X6wO/LljYO6LUDve6sRveTVEl8zbHlERrv98fHhTe4Z+sphRfZs7fT13TbFA8GZSZ+LfR/h4/naOpaRx4ky60xaLcbjjNn/LQebbZWm+XvwPSWP6c/k7f1KjQgRT7ruYCUv+4Z4ejQqccJl84izpmZnUqhjcSydrL5NShRVWFtpeD7dOhgfXWbfFMtNh+ihrYbAvBlpVkc/kLTNSnN3aOTbPtuiIUD66qT0RoSFMvMu9mewF+fqOCwt9Dl96ZcYmPpq3PXsezJXvLuDH5bsAqwO/y8t5V77Mbf/xszw8cRWv/raZ5f8cISU1g7ELdpDpouzOBc/PpNOLBZ830LSFopQvVaxjDT2+6N+QvAnWTLQek+625rc0vcy6Jda4D4QW71se0ZFhhJSR7PL13ZrE8PmwDj4596R7Omev1QLw/FWtePyntQBcEFuZpUnBsf5N37fms2HvcUbuPMrV8XXp/urcfPd/22Etl1N2MdDtyacY9cMathw4Sc0KkfRtnf9giGCmCUUpf4lpCj2fgB6Pw65EK7Gs+xHWT4KIitBigHVb7PzOAVm/Jct3/+rEpn3eVQbISihfDOtA1yYxPolnWOf6OZIJQKcG1kCBzo2q8ub17bjg+Zk+uVZhbXAYmPDunK0F7v/675uzn2clRcfRbjPW7aNv61ocOH6GhdsOYTCs3V18qjZocUililJGOuyYaw0/3jgZUk9CdC1odQ20HhTUJfad+XJREk/9so6tz/fzeORW7j6MsmEhpKRlsH70pZQLz/tZ1xiT3dfgTf9HcfHfy5ry8vRNTl8rzKixoigOqS0UpYpSSCg06m09Uk/D5mnW+i2LP4RF/7NGirW6xnpUd96hHUxu6hTLTZ1ifXKuK9rW4rvEXS4TU2mpFOwqmYA1TLpseOBaswXxW6e8iESKyBIRWSUi60Tk2Xz2HSQiRkT8mj2VCirh5azEMWQ8PLwZrngbKtaD+a/Bex3hvYuCogpyUXn+qtYse6I3EaH++4NZo0Lx7rdatSu4Kyb7s4VyFuhpjDkpImHAnyIyzRjzl+NOIhIN3Acs9mMsSgW3rBL77W+BE/utfpa1P1hVkGf/nzUMudU1VsHKCr5bzTGYhIWUoaqf5mbMfqgbx1LSqFO5LB2en+WXaxSFckHcOgE/JhRjdc6ctL8Msx/OOmz+D3gZeNhfsShVrETXgAuHW4+j/1hVkNf+AL89Dr89AedfZCWWFgOhvO/XOylJStqiWr6qMOAvfp2HIiIhIrISOAD8boxZnOv1dkA9Y8xkf8ahVLFV6TzofD/8ax7cuwx6PAanDsKUh+DVJvDlVdYcl5TgGEbriaw/ju/fGM8YN0qauGPds5cy/789ABjRvaHTZPLxzcX3znqFyLBAh5AvvyYUY0yGMSYOqAt0EJFWWa+JSBngDeChgs4jIsNFJFFEEpOTk/0XsFLBrFoj6PZfuGcxjFhoJZpD26w5Lq80gnGXw6L34PCOQEfqlmeuaAFAn5Y1GdzhvEKf76nLW1A+IpR6Vcqx9fm+/PdS50s7X1JAwcZJ93QudCz+snJncPehFNmwYRF5GjhljHnV/roisI1zt8VqAoeBAcYYl+OCddiwUg6Mgd3LrSHIm6ZBsr3+R0xzaNoXmvazFggrU7KKYjzzyzrGLUzil3s7Zxd09OT2lrNhx6ue7kPFsmFOX+/aJIZ5m4Pjw6y3Q4eL9ZryIhIjIpXs52WB3sDGrNeNMceMMdWMMbHGmFjgLwpIJkqpXESgbnvo/TTc8xfct8Jax6V8NVjwFnzaG15rCpPuhY1TraHKJcAzA1qy4slLaFO3ElXLhwOeDSte+dQlebaVz9XhfXW7OtnPvxjWgfWjL+XJy1t4HGvDmPIeH1Nc+XOUVy3gcxEJwUpc3xljJovIaCDRGPOLH6+tVOlUpQF0utt6pByBLTNh01Rr1NiKLyE0Ehr0sFovTS4r1oUrK9uJZOr9F/P3Ic8SZaVy4VwdX4dWtSsyevJ6IGeH94bRlxEeWoZbLorlWIq1/ny58NA8SccdM0d2497xK5jiwfLGxZXOlFeqNEhPhb8XWLfFNk2DY/Zqi3USrBIwcTfqiDE3zNqwn9s/z/n35+s7LuTGT1zPekga05+VO48y8F3r1pxjLbL1oy+lxVMzPIqhVN7yUkoFkdBwaNgD+r0MD6yGuxZAjyfAZMDvT8HrzeGHO+DvRVa/jHKqZ7PqjL31Ara90I8mNazVGzs3qpbjj/ywzvX57NYElj7emzXP9AGgtr1ezX8va8rEuy4CoN15lSgXHkq/1jWzj60QGZq9X3GkLRSlSrsDGyHxM2uhsLPHoXoLa9njNtdbyyArp46lpHHg+JnsBcSyOvIn/7sLrepUzLP/iTNpREWEIiIcPZ1KZFgIkWEhOY69Mq42k1buYebIrgx8dyEnz6bTrGY0G/edACA8pAybn+/rVbxF0ULRhKKUsqSesuqKJX4Ke1dBWHmrGvIFt0NN38wTKcl2H03hp+W73FowK7e40b9xR5f63N6lAQu3HaRX8xqcSctg8/4TtKlbiVNn0+n12h/MfKhbvgub5UcTihOaUJTys6yhyImfWjP0089A3Q5WYmkxEMIKv9ywKnqaUJzQhKJUETp92LoVlvgZHNoKZStbHfgJw6Bqw0BHpzygCcUJTShKBYAxsGOe1WrZMNnqzG/QAxJus0rxh5eeuRbFla6HopQKDiLQoJv1OL7XmtOybBx8d7M1t6V+V2hyKTS+FCrVC3S0KkC0haKU8k5GOvz9J2yeYc1tOWLXEKvRypo02eQyqBMf0OWN1Tl6y8sJTShKBSFj4OAW2DzdSjD/LLJui5WrBo37QNPLrFtkOgw5YPSWl1KqeBCBmCbWo/N9Vmf+ttlWgtk0FVZ9A2XCILbzudZLlfqBjlr5mLZQlFL+lZEOOxfbrZfpcHCztb1aU2jUy2q5xHbWjn0/01teTmhCUaqYO7QNtvxmJZe/F0HGWQgJh3oXWuVhGvSAWnElruR+oGlCcUITilIlSFqK1d+ybTZsmwv711jby1axRpQ17GklGB05Vmjah6KUKtnCylpJo2FP6+uTB2D7XDvBzIF1P1nbqza2Wi8Ne0JsF4iIDljIyjVtoSilgpMxkLzxXHJJ+hPSU6BMKNS9AM7raN0aq9UWKsdaAwOUS9pCUUqVXiJQvbn16HQPpJ+1Ove3zbZaMQvfgcx0a9/ISlZiqR1nJZnacVC5viaZIqYJRSlVPIRGWDPy63e1vk4/C/vXwd6VsGelVSF50XuQaa2wSGRFK8lktWJqt7OSjHb2+40mFKVU8RQaYc3ErxN/blt6KhxY75BkVsLiDyAj1Xo9ooKVXGq0gmqNoFoT6xFVQ1szPqAJRSlVcoSGW7e7asdBe3tbeiokb7BaMFlJZvkXkHbq3HHh0VCt8blH1cZWoqnSQMv1e0ATilKqZAsNt299tYX4m61txsDxPdYky0NbrX8PboakBbD623PHShmodN65BFOtkXXbrFwVq5R/2SrWhExt3QCaUJRSpZEIVKxjPRr2yPla6ik7yWyxH5vh0JZzo8xyKxNmJZfsJOPiUa6K1a8TGmlN5AwJs/91eF4mrFj38WhCUUopR+Hlz7VoHGVmwvFdcPQfSDmS83H68LnnR/+xbq+lHIG0055fv0yo64TT7RFoPcg336cfaEJRSil3lLFvf1U6z/1j0s7kTDxnjloDBDLS7H89fF62sv++Px/QhKKUUv4SFglhtaBCrUBHUiSK7806pZRSQUUTilJKKZ/QhKKUUsonNKEopZTyCb8lFBGJFJElIrJKRNaJyLNO9hkpIutFZLWIzBKR8/0Vj1JKKf/yZwvlLNDTGNMWiAMuE5GOufZZASQYY9oA3wMv+zEepZRSfuS3hGIsJ+0vw+yHybXPHGNM1syfv4C6/opHKaWUf/m1D0VEQkRkJXAA+N0Yszif3W8HpvkzHqWUUv7j14mNxpgMIE5EKgE/iUgrY8za3PuJyFAgAejm7DwiMhwYbn95UkQ2eRlSNeCgl8cWBY2vcDS+wtH4CifY42vq7wsU2RLAIvI0cMoY82qu7b2Bd4BuxpgDfo4h0d9LYBaGxlc4Gl/haHyFo/H5d5RXjN0yQUTKAr2Bjbn2aQd8CAzwdzJRSinlX/685VUL+FxEQrAS13fGmMkiMhpINMb8ArwCRAETxVpP4B9jzAA/xqSUUspP/JZQjDGrgXZOtj/l8Ly3v67vwkdFfD1PaXyFo/EVjsZXOKU+viLrQ1FKKVWyaekVpZRSvmGMKRUP4DJgE7AVGOXH69QD5gAbgHXA/fb2Z4DdwEr70c/hmEftuDYBlxYUM1AfWAxsAb4Fwj2MMQlYY8eRaG+rAvxun/N3oLK9XYC37RhWA/EO57nF3n8LcIvD9vb2+bfax4oHsTV1eI9WAseBBwL5/gGfYc2lWuuwze/vl6truBnfK1iDYFYDPwGV7O2xQIrD+/iBt3Hk9726EZ/ff55AhP31Vvv1WA/i+9YhtiRgZQDfP1d/U4LmdzD7PN780SxuDyAE2AY0AMKBVUALP12rVtYPEIgGNgMt7P9ADzvZv4UdT4T9H2ObHa/LmIHvgMH28w+AER7GmARUy7Xt5az/pMAo4CX7eT+sCacCdAQWO/yibbf/rWw/z/qFXgJ0so+ZBvQtxM9tH3B+IN8/oCsQT84/OH5/v1xdw834+gCh9vOXHOKLddwv13k8isPV9+pmfH7/eQJ3Y//BBwYD37obX67XXwOeCuD75+pvStD8DmbH6s1/9OL2sN+oGQ5fPwo8WkTXngRcks9/oByxADPseJ3GbP/AD3Luj0WO/dyMKYm8CWUTUMvhF3iT/fxDYEju/YAhwIcO2z+0t9UCNjpsz7Gfh3H2ARbYzwP6/pHrD0lRvF+uruFOfLleuwr4Or/9vInD1ffq5vvn959n1rH281B7P6et5XzeFwF2Ao0D+f7lulbW35Sg+h00xpSaPpQ6WL8UWXbZ2/xKRGKxRrpllZy5166s/JmIZC0O7So2V9urAkeNMem5tnvCAL+JyDK7CgFADWPMXgD73+pexlfHfp57uzcGA+Mdvg6W9w+K5v1ydQ1PDSNnWaP6IrJCRP4QkYsd4vY0jsL+v/L3zzP7GPv1Y/b+nrgY2G+M2eKwLWDvX66/KUH3O1haEoo42Wb8ekGRKOAH4AFjzHHgfaAhVuXlvVjN6Pxi83S7JzobY+KBvsA9ItI1n30DER8iEg4MACbam4Lp/ctPUMUjIo8D6cDX9qa9wHnGmHbASOAbEangZRyFib0ofp6+eG+HkPNDTcDePyd/Uzw9r99/B0tLQtmF1bGVpS6wx18XE5EwrB/818aYHwGMMfuNMRnGmEzgY6BDAbG52n4QqCQiobm2u80Ys8f+9wBWh20HYL+I1LLjr4XVSelNfLvIWTXa2/e6L7DcGLPfjjVo3j9bUbxfrq7hFhG5BbgcuNHY9yyMMWeNMYfs58uw+iWaeBmH1/+viujnmX2M/XpF4LA78TkcczVWB31W3AF5/5z9TfHivH7/HSwtCWUp0FhE6tuffAcDv/jjQmJN+f8U2GCMed1hey2H3a4Csopk/gIMFpEIEakPNMbqIHMas/2HYQ4wyD7+Fqx7qu7GV15EorOeY/VTrLXjuMXJOX8BbhZLR+CY3fSdAfQRkcr27Yo+WPeu9wInRKSj/V7c7El8DnJ8MgyW989BUbxfrq5RIBG5DHgEq6zRaYftMXb1CkSkAdb7td3LOFx9r+7EVxQ/T8e4BwGzsxKrm3pj9S1k3w4KxPvn6m+KF+f1/+9gQR1AJeWBNfJhM9Ynisf9eJ0uWM3F1TgMiQS+xBqWt9r+IdVyOOZxO65NOIyIchUz1kiXJVhD/CYCER7E1wBrhMwqrCGIj9vbqwKzsIYHzgKq2NsFeNeOYQ3WgmhZ5xpmx7AVuM1hewLWH4htwP/wYNiwfXw54BBQ0WFbwN4/rMS2F0jD+jR3e1G8X66u4WZ8W7Hul+cY3gpcY//cVwHLgSu8jSO/79WN+Pz+8wQi7a+32q83cDc+e/s44K5c+wbi/XP1NyVofgezHjpTXimllE+UllteSiml/EwTilJKKZ/QhKKUUsonNKEopZTyCU0oSimlfEITigo6IpIhIitFZJWILBeRiwrYv5KI3O3GeeeKSNCu+R0IIjJORAYVvKdSBdOEooJRijEmzhjTFqsA4IsF7F8Jq7JsUHKYxa1UiaYJRQW7CsARsGoZicgsu9WyRkSutPcZAzS0WzWv2Pv+195nlYiMcTjftSKyREQ2i13YT0RCROQVEVkqVrHCf9nba4nIPPu8a+VcIcBsIpIkIi/Z51wiIo3s7eNE5HURmQO8JCJVRORn+/x/iUgbh+9prB3rahG5xt7eR0QW2d/rRLHqOCEiY0Rkvb3vq/a2a+34VonIvAK+JxGR/9nnmIL3BSeVykM/OalgVFZEVmLNdK4F9LS3nwGuMsYcF5FqwF8i8gvWOg2tjDFxACLSFxgIXGiMOS0iVRzOHWqM6SAi/YCnscpr3I5Vns9wFJ8AAAL7SURBVOICEYkAFojIb1h1nGYYY563y22UcxHvcfucNwNvYtXPAqvGU29jTIaIvAOsMMYMFJGewBdYhRGftK/d2o69sv29PWEfe0pEHgFGisj/sMqUNDPGGBGpZF/nKayFqHY7bHP1PbXDWsSsNVADWI+1wJRShaYJRQWjFIfk0An4QkRaYZWUeEGs6siZWCW2azg5vjcw1tg1rIwxjgUBswrrLcNa2wKsmkZtHPoSKmLVaFoKfCZWYb6fjTErXcQ73uHfNxy2TzTGZNjPu2CV7cAYM1tEqopIRTvWwVkHGGOOiMjlWAsoLbBKKxEOLMJavfIM8InduphsH7YAGCci3zl8f66+p67AeDuuPSIy28X3pJTHNKGooGaMWWR/Yo/Bql8UA7Q3xqSJSBJWKyY3wXX57bP2vxmc+/0X4N/GmBl5TmQlr/7AlyLyijHmC2dhunh+KldMzo5zFqsAvxtjhjiJpwPQCysJ3Qv0NMbcJSIX2nGuFJE4V9+T3TLTekvKL7QPRQU1EWmGtfzrIaxP2QfsZNIDa2lggBNYS6Nm+Q0YJiLl7HM43vJyZgYwwm6JICJNxKrKfL59vY+xqr3Guzj+eod/F7nYZx5wo33+7sBBY61p8RtWYsj6fisDfwGdHfpjytkxRWEVzJwKPIB1ywwRaWiMWWyMeQqrnHs9V9+THcdgu4+lFtCjgPdGKbdpC0UFo6w+FLA+ad9i90N8DfwqIolYFVc3AhhjDonIAhFZC0wzxvzH/pSeKCKpwFTgsXyu9wnW7a/lYt1jSsbqg+kO/EdE0oCTWGW9nYkQkcVYH9DytCpszwBjRWQ1cJpzJcGfA961Y88AnjXG/CgitwLj7f4PsPpUTgCTRCTSfl8etF97RUQa29tmYVXCXe3ie/oJq09qDVbl3j/yeV+U8ohWG1aqEOzbbgnGmIOBjkWpQNNbXkoppXxCWyhKKaV8QlsoSimlfEITilJKKZ/QhKKUUsonNKEopZTyCU0oSimlfEITilJKKZ/4f1jUpczdPVjPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp.drop_mult=0.5\n",
    "learn=exp.create_cls_learner(data_clas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.true_wd=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load_encoder('enc_best')\n",
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 03:53\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.264060    0.178016    0.934240  (03:53)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 2e-2, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 04:21\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.261168    0.164097    0.940720  (04:21)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.freeze_to(-2)\n",
    "learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 05:39\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.222833    0.151485    0.944720  (05:39)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.freeze_to(-3)\n",
    "learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 13:48\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.210751    0.151522    0.948720  (06:54)\n",
      "2      0.181168    0.161364    0.948640  (06:53)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('best_cls')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load(\"best_cls\");"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:fastaiv1]",
   "language": "python",
   "name": "conda-env-fastaiv1-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}