diff --git a/notebooks/JA-multifit_fp16.ipynb b/notebooks/JA-multifit_fp16.ipynb index b214f97..690f8b9 100644 --- a/notebooks/JA-multifit_fp16.ipynb +++ b/notebooks/JA-multifit_fp16.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [ { @@ -19,7 +19,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -40,12 +40,12 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "from ulmfit.datasets import ULMFiTDataset, Dataset\n", - "from ulmfit import ULMFiT, multifit1552_fp16" + "from ulmfit import ULMFiT, multifit1552_fp16, multifit1552_fp32" ] }, { @@ -57,7 +57,55 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 4, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "def multifit1552_fp16():\n", + " return multifit1552_fp32(bs=128).replace_(fp16=True, name=multifit1552_fp16.__name__)\n", + "\n", + "def multifit1552_fp32(bs=64):\n", + " self = ULMFiT()\n", + " self.replace_(\n", + " label_smoothing_eps=0.0,\n", + " true_wd=True,\n", + " wd=0.1,\n", + " seed=0,\n", + " fp16=False,\n", + " bs=bs,\n", + " use_adam_08=False,\n", + " early_stopping=None,\n", + " name=multifit1552_fp32.__name__\n", + " )\n", + " self.arch.replace_(\n", + " tokenizer='fsp',\n", + " max_vocab=15000,\n", + " qrnn=True,\n", + " n_layers=4,\n", + " n_hid=1552\n", + " )\n", + " self.pretrain_lm.replace_(num_epochs=10, drop_mult=0.5, lr=(1e-2 * bs / 48))\n", + " self.finetuine_lm.replace_(num_epochs=10, drop_mult=1.0, lr=(1e-3 * bs / 48))\n", + " self.classifier.replace_(num_epochs=8, drop_mult=0.5, bs=20, label_smoothing_eps=0.1)\n", + " return self\n", + "\n" + ] + } + ], + "source": [ + "import inspect\n", + "print(inspect.getsource(multifit1552_fp16))\n", + "print(inspect.getsource(multifit1552_fp32))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -66,7 +114,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -75,7 +123,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { diff --git a/ulmfit/training.py b/ulmfit/training.py index 010d659..94d1daa 100644 --- a/ulmfit/training.py +++ b/ulmfit/training.py @@ -454,3 +454,11 @@ class ULMFiT: self.finetuine_lm.replace_(**kwargs) self.classifier.replace_(**kwargs) return self + + def pprint(self): + print(f"""ULMFiT( + {self.arch}, + {self.pretrain_lm}, + {self.finetuine_lm}, + {self.classifier}, +)""")