wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-11 12:20:41 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
master
multifit/notebooks/CLS-JA.ipynb
T

1787 lines
69 KiB
Plaintext
Raw Permalink Blame History

{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"env: CUDA_VISIBLE_DEVICES=0\n"
]
}
],
"source": [
"%env CUDA_VISIBLE_DEVICES=0"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/home/pczapla/workspace/ulmfit-multilingual\n"
]
}
],
"source": [
"%reload_ext autoreload\n",
"%autoreload 2\n",
"%matplotlib inline\n",
"%cd .."
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"from fastai.text import *\n",
"from multifit.datasets import ULMFiTDataset, Dataset\n",
"import multifit"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"import warnings\n",
"warnings.filterwarnings(\"ignore\") # to ignore pytorch 1.3 warnings triggered by older fastai"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# CLS JA music"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"#!python prepare_cls.py https://storage.googleapis.com/ulmfit/cls"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"exp = multifit.from_pretrained('ja_multifit_paper_version')"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/plain": [
"ULMFiTArchitecture(tokenizer_type='sp', max_vocab=15000, lang='ja', emb_sz=400, n_hid=1550, n_layers=4, qrnn=True)"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"exp.arch"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Copy sp model from /home/pczapla/.fastai/models/ja_multifit_paper_version to data/cls/ja-music/models/sp15k\n"
]
}
],
"source": [
"cls_dataset = exp.arch.dataset(Path('data/cls/ja-music'), exp.pretrain_lm.tokenizer)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'ja'"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cls_dataset.lang"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Data tst, trn: 200, val: 1999\n"
]
},
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th>text</th>\n",
" <th>target</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>▁ xxbos ▁ xxfld ▁ 1 ▁一 音 が すべて 、 すべてが 一 音 。 &lt;unk&gt; し み を 包 んで く れる 宇 宙 その もの ▁ xxfld ▁ 2 ▁ xxup ▁ &lt;unk&gt; かな 世 界 が 、 cd を かけた と たん 拡 が る 。 「 打 波 」 だ 。 それは &lt;unk&gt; 気 と &lt;unk&gt; 発 力 をもって はじまり 、 美 しい 小 波 が</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>▁ xxbos ▁ xxfld ▁ 1 ▁ 様 々 な バージョン 違 い や レア 音 源 を 聴 き 散 ら か した 上 で 聴 く と 、 どう にも &lt;unk&gt; 求 不 満 になる アルバム 。 イン スト 部 分 が 多 い のも 「 歌 」 を 充 分 聴 きた い 人 間 には 物 足 り ない かな 、 と 。 ▁ xxfld ▁</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>▁ xxbos ▁ xxfld ▁ 1 ▁ xxup ▁ 今 度 は 野 球 の 松 坂 ファンを ex ファンに させ ようと してる ▁ xxrep ▁4 ▁・ ▁ xxfld ▁ 2 ▁ xxup ▁第 2 章 になって すべての シングル 、 アルバムの dvd 付 き を 購 入 した 方 。 よく 考 え ましょう 。 ▁ xxup ▁ cd 収 録 の ▁ xxup ▁ w on ▁ ' ▁</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>▁ xxbos ▁ xxfld ▁ 1 ▁ 粒 &lt;unk&gt; い の light な d ance ▁number が &lt;unk&gt; 能 出 来 る b est 盤 ▁ xxfld ▁ 2 ▁ sky y は 80 年 代 を pe ak に 大 活 躍 した new - y or k のお &lt;unk&gt; 落 な fu nk ▁ xxmaj ▁ band である 。 ▁ただ 、 fu nk ▁ band とは 言 え</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>▁ xxbos ▁ xxfld ▁ 1 ▁ 少 々 物 足 り ない 内 容 ▁ xxfld ▁ 2 ▁ レイ ・ デイヴィス 自 身 が 監 修 した キン クス 待 望 の ボックス ・ セット の 登 場 です 。 p ye 〜 rca 〜 ari sta 〜 lon don 〜 co lum bia 〜 ko nk の 全 時 代 を 網 羅 した 6 枚 組</td>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"cls_dataset.load_clas_databunch(bs=exp.finetune_lm.bs).show_batch()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Setting LM weights seed seed to 0\n",
"Data lm-notst, trn: 31800, val: 200\n",
"Size of vocabulary: 15000\n",
"First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
"Training args: {'drop_mult': 0.3, 'true_wd': False, 'wd': 1e-07, 'pretrained': False, 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15, 'tie_weights': True, 'out_bias': True}\n",
"Setting LM training seed seed to 0\n",
"Loading pretrained weights: [PosixPath('/home/pczapla/.fastai/models/ja_multifit_paper_version/lm_best'), PosixPath('/home/pczapla/.fastai/models/ja_multifit_paper_version/itos')]\n",
"Experiment data/cls/ja-music/models/sp15k/multifit_paper_version\n",
"Fitting using 2 cycle fit schedule\n"
]
},
{
"data": {
"text/html": [
"Total time: 02:14 <p><table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>5.815562</td>\n",
" <td>5.125886</td>\n",
" <td>0.214914</td>\n",
" <td>02:13</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Total time: 1:04:16 <p><table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>4.962217</td>\n",
" <td>4.831137</td>\n",
" <td>0.238743</td>\n",
" <td>03:10</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>4.559075</td>\n",
" <td>4.432836</td>\n",
" <td>0.278943</td>\n",
" <td>03:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>4.131804</td>\n",
" <td>4.128801</td>\n",
" <td>0.312514</td>\n",
" <td>03:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>3.960420</td>\n",
" <td>3.925446</td>\n",
" <td>0.332600</td>\n",
" <td>03:13</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>3.833594</td>\n",
" <td>3.798564</td>\n",
" <td>0.346086</td>\n",
" <td>03:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>3.664375</td>\n",
" <td>3.708547</td>\n",
" <td>0.354543</td>\n",
" <td>03:13</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>3.592303</td>\n",
" <td>3.634840</td>\n",
" <td>0.362600</td>\n",
" <td>03:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>3.448824</td>\n",
" <td>3.587191</td>\n",
" <td>0.367714</td>\n",
" <td>03:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>8</td>\n",
" <td>3.456172</td>\n",
" <td>3.549420</td>\n",
" <td>0.374629</td>\n",
" <td>03:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>9</td>\n",
" <td>3.353690</td>\n",
" <td>3.518477</td>\n",
" <td>0.378029</td>\n",
" <td>03:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>10</td>\n",
" <td>3.329306</td>\n",
" <td>3.500824</td>\n",
" <td>0.381371</td>\n",
" <td>03:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>11</td>\n",
" <td>3.270015</td>\n",
" <td>3.491282</td>\n",
" <td>0.382200</td>\n",
" <td>03:14</td>\n",
" </tr>\n",
" <tr>\n",
" <td>12</td>\n",
" <td>3.166214</td>\n",
" <td>3.475223</td>\n",
" <td>0.384343</td>\n",
" <td>03:13</td>\n",
" </tr>\n",
" <tr>\n",
" <td>13</td>\n",
" <td>3.155823</td>\n",
" <td>3.467666</td>\n",
" <td>0.387829</td>\n",
" <td>03:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>14</td>\n",
" <td>3.136682</td>\n",
" <td>3.457926</td>\n",
" <td>0.389714</td>\n",
" <td>03:13</td>\n",
" </tr>\n",
" <tr>\n",
" <td>15</td>\n",
" <td>3.022880</td>\n",
" <td>3.463706</td>\n",
" <td>0.388857</td>\n",
" <td>03:13</td>\n",
" </tr>\n",
" <tr>\n",
" <td>16</td>\n",
" <td>3.058692</td>\n",
" <td>3.461311</td>\n",
" <td>0.389086</td>\n",
" <td>03:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>17</td>\n",
" <td>3.008492</td>\n",
" <td>3.461708</td>\n",
" <td>0.388857</td>\n",
" <td>03:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>18</td>\n",
" <td>2.982608</td>\n",
" <td>3.462014</td>\n",
" <td>0.389171</td>\n",
" <td>03:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>19</td>\n",
" <td>2.988810</td>\n",
" <td>3.461781</td>\n",
" <td>0.389457</td>\n",
" <td>03:13</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Copy sp model from /home/pczapla/.fastai/models/ja_multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_version\n",
"this Learner object self-destroyed - it still exists, but no longer usable\n",
"Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
"Language model saved to data/cls/ja-music/models/sp15k/multifit_paper_version\n"
]
}
],
"source": [
"exp.finetune_lm.train_(cls_dataset)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Loading data/cls/ja-music/models/sp15k/multifit_paper_version/classifier.json\n",
"Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
"ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
"ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
"ULMFiTClassifier Replacing bs '18' with '20\n",
"ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
"ULMFiTClassifier Replacing early_stopping 'None' with 'accuracy\n"
]
},
{
"data": {
"text/plain": [
"ULMFiTFinetuning(seed=0, name='multifit_paper_version', experiment_path=PosixPath('data/cls/ja-music/models/sp15k/multifit_paper_version'), dataset_path=PosixPath('data/cls/ja-music'), num_epochs=20, bs=20, bptt=70, drop_mult=0.3, dropout_values={'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}, label_smoothing_eps=0.0, label_smoothing_eps_norm_by_classes=True, use_adam_08=False, true_wd=False, wd=1e-07, clip=0.12, fp16=False, lr=0.001)"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"exp.load_(cls_dataset.cache_path/exp.pretrain_lm.name).finetune_lm"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
"Data lm-notst, trn: 31800, val: 200\n",
"Size of vocabulary: 15000\n",
"First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
"Data cls, trn: 1800, val: 200\n",
"Data tst, trn: 200, val: 1999\n",
"Using Label smoothing with eps = 0.05\n",
"Setting Classifier weights seed seed to 0\n",
"Training args: {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
"Setting Classifier training seed seed to 0\n",
"Training: data/cls/ja-music/models/sp15k/multifit_paper_version\n"
]
},
{
"data": {
"text/html": [
"Total time: 01:38 <p><table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0.551692</td>\n",
" <td>0.544244</td>\n",
" <td>0.750000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>0.514899</td>\n",
" <td>0.617670</td>\n",
" <td>0.750000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>0.443367</td>\n",
" <td>0.596144</td>\n",
" <td>0.785000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>0.368091</td>\n",
" <td>0.567668</td>\n",
" <td>0.790000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>0.315757</td>\n",
" <td>0.635034</td>\n",
" <td>0.805000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>0.271595</td>\n",
" <td>0.515227</td>\n",
" <td>0.815000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>0.241717</td>\n",
" <td>0.523387</td>\n",
" <td>0.820000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>0.237796</td>\n",
" <td>0.518763</td>\n",
" <td>0.825000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Better model found at epoch 0 with accuracy value: 0.75.\n",
"Better model found at epoch 2 with accuracy value: 0.7850000262260437.\n",
"Better model found at epoch 3 with accuracy value: 0.7900000214576721.\n",
"Better model found at epoch 4 with accuracy value: 0.8050000071525574.\n",
"Better model found at epoch 5 with accuracy value: 0.8149999976158142.\n",
"Better model found at epoch 6 with accuracy value: 0.8199999928474426.\n",
"Better model found at epoch 7 with accuracy value: 0.824999988079071.\n",
"Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_version\n",
"Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_version/classifier.json\n",
"this Learner object self-destroyed - it still exists, but no longer usable\n"
]
}
],
"source": [
"exp.classifier.train_(seed=0)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
"Data lm-notst, trn: 31800, val: 200\n",
"Size of vocabulary: 15000\n",
"First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
"Data cls, trn: 1800, val: 200\n",
"Data tst, trn: 200, val: 1999\n",
"Using Label smoothing with eps = 0.05\n",
"Setting Classifier weights seed seed to 1\n",
"Training args: {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
"Setting Classifier training seed seed to 1\n",
"Training: data/cls/ja-music/models/sp15k/multifit_paper_versionseed1\n"
]
},
{
"data": {
"text/html": [
"Total time: 01:35 <p><table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0.553418</td>\n",
" <td>0.493184</td>\n",
" <td>0.800000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>0.517754</td>\n",
" <td>0.603011</td>\n",
" <td>0.745000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>0.444649</td>\n",
" <td>0.451021</td>\n",
" <td>0.850000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>0.378163</td>\n",
" <td>0.488551</td>\n",
" <td>0.845000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>0.296332</td>\n",
" <td>0.722608</td>\n",
" <td>0.745000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>0.267031</td>\n",
" <td>0.528215</td>\n",
" <td>0.815000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>0.248884</td>\n",
" <td>0.522899</td>\n",
" <td>0.830000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>0.234614</td>\n",
" <td>0.512867</td>\n",
" <td>0.825000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Better model found at epoch 0 with accuracy value: 0.800000011920929.\n",
"Better model found at epoch 2 with accuracy value: 0.8500000238418579.\n",
"Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_versionseed1\n",
"Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_versionseed1\n",
"Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_versionseed1/classifier.json\n",
"this Learner object self-destroyed - it still exists, but no longer usable\n"
]
}
],
"source": [
"exp.classifier.train_(seed=1)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
"Data lm-notst, trn: 31800, val: 200\n",
"Size of vocabulary: 15000\n",
"First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
"Data cls, trn: 1800, val: 200\n",
"Data tst, trn: 200, val: 1999\n",
"Using Label smoothing with eps = 0.05\n",
"Setting Classifier weights seed seed to 2\n",
"Training args: {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
"Setting Classifier training seed seed to 2\n",
"Training: data/cls/ja-music/models/sp15k/multifit_paper_versionseed2\n"
]
},
{
"data": {
"text/html": [
"Total time: 01:35 <p><table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0.522099</td>\n",
" <td>0.587937</td>\n",
" <td>0.780000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>0.501093</td>\n",
" <td>0.749337</td>\n",
" <td>0.755000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>0.431629</td>\n",
" <td>0.610693</td>\n",
" <td>0.765000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>0.369956</td>\n",
" <td>0.518467</td>\n",
" <td>0.820000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>0.300080</td>\n",
" <td>0.621023</td>\n",
" <td>0.810000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>0.273065</td>\n",
" <td>0.548695</td>\n",
" <td>0.810000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>0.244343</td>\n",
" <td>0.521141</td>\n",
" <td>0.820000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>0.228544</td>\n",
" <td>0.508984</td>\n",
" <td>0.820000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Better model found at epoch 0 with accuracy value: 0.7799999713897705.\n",
"Better model found at epoch 3 with accuracy value: 0.8199999928474426.\n",
"Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_versionseed2\n",
"Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_versionseed2\n",
"Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_versionseed2/classifier.json\n",
"this Learner object self-destroyed - it still exists, but no longer usable\n"
]
}
],
"source": [
"exp.classifier.train_(seed=2)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
"Data lm-notst, trn: 31800, val: 200\n",
"Size of vocabulary: 15000\n",
"First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
"Data cls, trn: 1800, val: 200\n",
"Data tst, trn: 200, val: 1999\n",
"Using Label smoothing with eps = 0.05\n",
"Setting Classifier weights seed seed to 3\n",
"Training args: {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
"Setting Classifier training seed seed to 3\n",
"Training: data/cls/ja-music/models/sp15k/multifit_paper_versionseed3\n"
]
},
{
"data": {
"text/html": [
"Total time: 01:36 <p><table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0.540789</td>\n",
" <td>0.530250</td>\n",
" <td>0.775000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>0.494581</td>\n",
" <td>0.509151</td>\n",
" <td>0.815000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>0.446126</td>\n",
" <td>0.529051</td>\n",
" <td>0.785000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>0.376928</td>\n",
" <td>0.582771</td>\n",
" <td>0.775000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>0.322823</td>\n",
" <td>0.492915</td>\n",
" <td>0.835000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>0.270082</td>\n",
" <td>0.499838</td>\n",
" <td>0.850000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>0.249619</td>\n",
" <td>0.512006</td>\n",
" <td>0.840000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>0.240476</td>\n",
" <td>0.515156</td>\n",
" <td>0.820000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Better model found at epoch 0 with accuracy value: 0.7749999761581421.\n",
"Better model found at epoch 1 with accuracy value: 0.8149999976158142.\n",
"Better model found at epoch 4 with accuracy value: 0.8349999785423279.\n",
"Better model found at epoch 5 with accuracy value: 0.8500000238418579.\n",
"Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_versionseed3\n",
"Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_versionseed3\n",
"Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_versionseed3/classifier.json\n",
"this Learner object self-destroyed - it still exists, but no longer usable\n"
]
}
],
"source": [
"exp.classifier.train_(seed=3)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
"Data lm-notst, trn: 31800, val: 200\n",
"Size of vocabulary: 15000\n",
"First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
"Data cls, trn: 1800, val: 200\n",
"Data tst, trn: 200, val: 1999\n",
"Using Label smoothing with eps = 0.05\n",
"Setting Classifier weights seed seed to 4\n",
"Training args: {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
"Setting Classifier training seed seed to 4\n",
"Training: data/cls/ja-music/models/sp15k/multifit_paper_versionseed4\n"
]
},
{
"data": {
"text/html": [
"Total time: 01:38 <p><table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0.557735</td>\n",
" <td>0.474914</td>\n",
" <td>0.805000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>0.497810</td>\n",
" <td>0.519276</td>\n",
" <td>0.785000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>0.445226</td>\n",
" <td>0.632691</td>\n",
" <td>0.775000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>0.376329</td>\n",
" <td>0.523593</td>\n",
" <td>0.810000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>0.315562</td>\n",
" <td>0.491826</td>\n",
" <td>0.815000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>0.277676</td>\n",
" <td>0.531970</td>\n",
" <td>0.805000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>0.241653</td>\n",
" <td>0.491889</td>\n",
" <td>0.840000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>0.240878</td>\n",
" <td>0.505452</td>\n",
" <td>0.835000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Better model found at epoch 0 with accuracy value: 0.8050000071525574.\n",
"Better model found at epoch 3 with accuracy value: 0.8100000023841858.\n",
"Better model found at epoch 4 with accuracy value: 0.8149999976158142.\n",
"Better model found at epoch 6 with accuracy value: 0.8399999737739563.\n",
"Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_versionseed4\n",
"Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_versionseed4\n",
"Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_versionseed4/classifier.json\n",
"this Learner object self-destroyed - it still exists, but no longer usable\n"
]
}
],
"source": [
"exp.classifier.train_(seed=4)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
"Data lm-notst, trn: 31800, val: 200\n",
"Size of vocabulary: 15000\n",
"First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
"Data cls, trn: 1800, val: 200\n",
"Data tst, trn: 200, val: 1999\n",
"Using Label smoothing with eps = 0.05\n",
"Setting Classifier weights seed seed to 5\n",
"Training args: {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
"Setting Classifier training seed seed to 5\n",
"Training: data/cls/ja-music/models/sp15k/multifit_paper_versionseed5\n"
]
},
{
"data": {
"text/html": [
"Total time: 01:35 <p><table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0.539067</td>\n",
" <td>0.524193</td>\n",
" <td>0.820000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>0.504846</td>\n",
" <td>0.563058</td>\n",
" <td>0.775000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>0.455630</td>\n",
" <td>0.554939</td>\n",
" <td>0.785000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>0.366007</td>\n",
" <td>0.535870</td>\n",
" <td>0.810000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>0.302207</td>\n",
" <td>0.502417</td>\n",
" <td>0.825000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>0.274910</td>\n",
" <td>0.543057</td>\n",
" <td>0.815000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>0.257135</td>\n",
" <td>0.517289</td>\n",
" <td>0.815000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>0.234617</td>\n",
" <td>0.505733</td>\n",
" <td>0.800000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Better model found at epoch 0 with accuracy value: 0.8199999928474426.\n",
"Better model found at epoch 4 with accuracy value: 0.824999988079071.\n",
"Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_versionseed5\n",
"Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_versionseed5\n",
"Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_versionseed5/classifier.json\n",
"this Learner object self-destroyed - it still exists, but no longer usable\n"
]
}
],
"source": [
"exp.classifier.train_(seed=5)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
"Data lm-notst, trn: 31800, val: 200\n",
"Size of vocabulary: 15000\n",
"First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
"Data cls, trn: 1800, val: 200\n",
"Data tst, trn: 200, val: 1999\n",
"Using Label smoothing with eps = 0.05\n",
"Setting Classifier weights seed seed to 6\n",
"Training args: {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
"Setting Classifier training seed seed to 6\n",
"Training: data/cls/ja-music/models/sp15k/multifit_paper_versionseed6\n"
]
},
{
"data": {
"text/html": [
"Total time: 01:38 <p><table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0.529963</td>\n",
" <td>0.533260</td>\n",
" <td>0.750000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>0.501183</td>\n",
" <td>0.518192</td>\n",
" <td>0.785000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>0.429616</td>\n",
" <td>0.633020</td>\n",
" <td>0.800000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>0.382384</td>\n",
" <td>0.526554</td>\n",
" <td>0.830000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>0.315402</td>\n",
" <td>0.520821</td>\n",
" <td>0.825000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>0.274772</td>\n",
" <td>0.551267</td>\n",
" <td>0.820000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>0.249956</td>\n",
" <td>0.519960</td>\n",
" <td>0.830000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>0.226764</td>\n",
" <td>0.516419</td>\n",
" <td>0.825000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Better model found at epoch 0 with accuracy value: 0.75.\n",
"Better model found at epoch 1 with accuracy value: 0.7850000262260437.\n",
"Better model found at epoch 2 with accuracy value: 0.800000011920929.\n",
"Better model found at epoch 3 with accuracy value: 0.8299999833106995.\n",
"Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_versionseed6\n",
"Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_versionseed6\n",
"Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_versionseed6/classifier.json\n",
"this Learner object self-destroyed - it still exists, but no longer usable\n"
]
}
],
"source": [
"exp.classifier.train_(seed=6)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
"Data lm-notst, trn: 31800, val: 200\n",
"Size of vocabulary: 15000\n",
"First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
"Data cls, trn: 1800, val: 200\n",
"Data tst, trn: 200, val: 1999\n",
"Using Label smoothing with eps = 0.05\n",
"Setting Classifier weights seed seed to 7\n",
"Training args: {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
"Setting Classifier training seed seed to 7\n",
"Training: data/cls/ja-music/models/sp15k/multifit_paper_versionseed7\n"
]
},
{
"data": {
"text/html": [
"Total time: 01:36 <p><table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0.565436</td>\n",
" <td>0.517827</td>\n",
" <td>0.805000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>0.517789</td>\n",
" <td>0.710051</td>\n",
" <td>0.795000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>0.456400</td>\n",
" <td>0.502067</td>\n",
" <td>0.805000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>0.367990</td>\n",
" <td>0.560270</td>\n",
" <td>0.800000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>0.305195</td>\n",
" <td>0.585719</td>\n",
" <td>0.805000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>0.276869</td>\n",
" <td>0.496807</td>\n",
" <td>0.840000</td>\n",
" <td>00:12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>0.244716</td>\n",
" <td>0.502482</td>\n",
" <td>0.845000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>0.238059</td>\n",
" <td>0.496951</td>\n",
" <td>0.845000</td>\n",
" <td>00:11</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Better model found at epoch 0 with accuracy value: 0.8050000071525574.\n",
"Better model found at epoch 5 with accuracy value: 0.8399999737739563.\n",
"Better model found at epoch 6 with accuracy value: 0.8450000286102295.\n",
"Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_versionseed7\n",
"Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_versionseed7\n",
"Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_versionseed7/classifier.json\n",
"this Learner object self-destroyed - it still exists, but no longer usable\n"
]
}
],
"source": [
"exp.classifier.train_(seed=7)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Results"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Loading data/cls/ja-music/models/sp15k/multifit_paper_versionseed5/classifier.json\n",
"Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
"ULMFiTFinetuning Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
"ULMFiTFinetuning Replacing dataset_path 'None' with 'data/cls/ja-music\n",
"ULMFiTFinetuning Replacing num_epochs '10' with '20\n",
"ULMFiTFinetuning Replacing drop_mult '1.0' with '0.3\n",
"ULMFiTFinetuning Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"ULMFiTFinetuning Replacing true_wd 'True' with 'False\n",
"ULMFiTFinetuning Replacing wd '0.01' with '1e-07\n",
"ULMFiTFinetuning Replacing clip 'None' with '0.12\n",
"ULMFiTArchitecture Replacing tokenizer_type 'f' with 'sp\n",
"ULMFiTArchitecture Replacing max_vocab '60000' with '15000\n",
"ULMFiTArchitecture Replacing lang 'None' with 'ja\n",
"ULMFiTArchitecture Replacing n_hid '1150' with '1550\n",
"ULMFiTArchitecture Replacing n_layers '3' with '4\n",
"ULMFiTArchitecture Replacing qrnn 'False' with 'True\n",
"ULMFiTClassifier Replacing seed '0' with '5\n",
"ULMFiTClassifier Replacing name 'multifit_paper_versionseed5' with 'multifit_paper_version\n",
"ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_versionseed5\n",
"ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
"ULMFiTClassifier Replacing num_epochs '10' with '8\n",
"ULMFiTClassifier Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
"ULMFiTClassifier Replacing clip 'None' with '0.12\n",
"ULMFiTClassifier Replacing label_smoothing_eps '0.0' with '0.1\n",
"ULMFiTClassifier Replacing label_smoothing_eps_norm_by_classes 'False' with 'True\n",
"Loading data/cls/ja-music/models/sp15k/multifit_paper_versionseed4/classifier.json\n",
"Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
"ULMFiTFinetuning Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
"ULMFiTFinetuning Replacing dataset_path 'None' with 'data/cls/ja-music\n",
"ULMFiTFinetuning Replacing num_epochs '10' with '20\n",
"ULMFiTFinetuning Replacing drop_mult '1.0' with '0.3\n",
"ULMFiTFinetuning Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"ULMFiTFinetuning Replacing true_wd 'True' with 'False\n",
"ULMFiTFinetuning Replacing wd '0.01' with '1e-07\n",
"ULMFiTFinetuning Replacing clip 'None' with '0.12\n",
"ULMFiTArchitecture Replacing tokenizer_type 'f' with 'sp\n",
"ULMFiTArchitecture Replacing max_vocab '60000' with '15000\n",
"ULMFiTArchitecture Replacing lang 'None' with 'ja\n",
"ULMFiTArchitecture Replacing n_hid '1150' with '1550\n",
"ULMFiTArchitecture Replacing n_layers '3' with '4\n",
"ULMFiTArchitecture Replacing qrnn 'False' with 'True\n",
"ULMFiTClassifier Replacing seed '0' with '4\n",
"ULMFiTClassifier Replacing name 'multifit_paper_versionseed4' with 'multifit_paper_version\n",
"ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_versionseed4\n",
"ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
"ULMFiTClassifier Replacing num_epochs '10' with '8\n",
"ULMFiTClassifier Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
"ULMFiTClassifier Replacing clip 'None' with '0.12\n",
"ULMFiTClassifier Replacing label_smoothing_eps '0.0' with '0.1\n",
"ULMFiTClassifier Replacing label_smoothing_eps_norm_by_classes 'False' with 'True\n",
"Loading data/cls/ja-music/models/sp15k/multifit_paper_versionseed3/classifier.json\n",
"Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
"ULMFiTFinetuning Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
"ULMFiTFinetuning Replacing dataset_path 'None' with 'data/cls/ja-music\n",
"ULMFiTFinetuning Replacing num_epochs '10' with '20\n",
"ULMFiTFinetuning Replacing drop_mult '1.0' with '0.3\n",
"ULMFiTFinetuning Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"ULMFiTFinetuning Replacing true_wd 'True' with 'False\n",
"ULMFiTFinetuning Replacing wd '0.01' with '1e-07\n",
"ULMFiTFinetuning Replacing clip 'None' with '0.12\n",
"ULMFiTArchitecture Replacing tokenizer_type 'f' with 'sp\n",
"ULMFiTArchitecture Replacing max_vocab '60000' with '15000\n",
"ULMFiTArchitecture Replacing lang 'None' with 'ja\n",
"ULMFiTArchitecture Replacing n_hid '1150' with '1550\n",
"ULMFiTArchitecture Replacing n_layers '3' with '4\n",
"ULMFiTArchitecture Replacing qrnn 'False' with 'True\n",
"ULMFiTClassifier Replacing seed '0' with '3\n",
"ULMFiTClassifier Replacing name 'multifit_paper_versionseed3' with 'multifit_paper_version\n",
"ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_versionseed3\n",
"ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
"ULMFiTClassifier Replacing num_epochs '10' with '8\n",
"ULMFiTClassifier Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
"ULMFiTClassifier Replacing clip 'None' with '0.12\n",
"ULMFiTClassifier Replacing label_smoothing_eps '0.0' with '0.1\n",
"ULMFiTClassifier Replacing label_smoothing_eps_norm_by_classes 'False' with 'True\n",
"Loading data/cls/ja-music/models/sp15k/multifit_paper_versionseed1/classifier.json\n",
"Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
"ULMFiTFinetuning Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
"ULMFiTFinetuning Replacing dataset_path 'None' with 'data/cls/ja-music\n",
"ULMFiTFinetuning Replacing num_epochs '10' with '20\n",
"ULMFiTFinetuning Replacing drop_mult '1.0' with '0.3\n",
"ULMFiTFinetuning Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"ULMFiTFinetuning Replacing true_wd 'True' with 'False\n",
"ULMFiTFinetuning Replacing wd '0.01' with '1e-07\n",
"ULMFiTFinetuning Replacing clip 'None' with '0.12\n",
"ULMFiTArchitecture Replacing tokenizer_type 'f' with 'sp\n",
"ULMFiTArchitecture Replacing max_vocab '60000' with '15000\n",
"ULMFiTArchitecture Replacing lang 'None' with 'ja\n",
"ULMFiTArchitecture Replacing n_hid '1150' with '1550\n",
"ULMFiTArchitecture Replacing n_layers '3' with '4\n",
"ULMFiTArchitecture Replacing qrnn 'False' with 'True\n",
"ULMFiTClassifier Replacing seed '0' with '1\n",
"ULMFiTClassifier Replacing name 'multifit_paper_versionseed1' with 'multifit_paper_version\n",
"ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_versionseed1\n",
"ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
"ULMFiTClassifier Replacing num_epochs '10' with '8\n",
"ULMFiTClassifier Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
"ULMFiTClassifier Replacing clip 'None' with '0.12\n",
"ULMFiTClassifier Replacing label_smoothing_eps '0.0' with '0.1\n",
"ULMFiTClassifier Replacing label_smoothing_eps_norm_by_classes 'False' with 'True\n",
"Loading data/cls/ja-music/models/sp15k/multifit_paper_versionseed2/classifier.json\n",
"Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
"ULMFiTFinetuning Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
"ULMFiTFinetuning Replacing dataset_path 'None' with 'data/cls/ja-music\n",
"ULMFiTFinetuning Replacing num_epochs '10' with '20\n",
"ULMFiTFinetuning Replacing drop_mult '1.0' with '0.3\n",
"ULMFiTFinetuning Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"ULMFiTFinetuning Replacing true_wd 'True' with 'False\n",
"ULMFiTFinetuning Replacing wd '0.01' with '1e-07\n",
"ULMFiTFinetuning Replacing clip 'None' with '0.12\n",
"ULMFiTArchitecture Replacing tokenizer_type 'f' with 'sp\n",
"ULMFiTArchitecture Replacing max_vocab '60000' with '15000\n",
"ULMFiTArchitecture Replacing lang 'None' with 'ja\n",
"ULMFiTArchitecture Replacing n_hid '1150' with '1550\n",
"ULMFiTArchitecture Replacing n_layers '3' with '4\n",
"ULMFiTArchitecture Replacing qrnn 'False' with 'True\n",
"ULMFiTClassifier Replacing seed '0' with '2\n",
"ULMFiTClassifier Replacing name 'multifit_paper_versionseed2' with 'multifit_paper_version\n",
"ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_versionseed2\n",
"ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
"ULMFiTClassifier Replacing num_epochs '10' with '8\n",
"ULMFiTClassifier Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
"ULMFiTClassifier Replacing clip 'None' with '0.12\n",
"ULMFiTClassifier Replacing label_smoothing_eps '0.0' with '0.1\n",
"ULMFiTClassifier Replacing label_smoothing_eps_norm_by_classes 'False' with 'True\n",
"Loading data/cls/ja-music/models/sp15k/multifit_paper_versionseed6/classifier.json\n",
"Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
"ULMFiTFinetuning Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
"ULMFiTFinetuning Replacing dataset_path 'None' with 'data/cls/ja-music\n",
"ULMFiTFinetuning Replacing num_epochs '10' with '20\n",
"ULMFiTFinetuning Replacing drop_mult '1.0' with '0.3\n",
"ULMFiTFinetuning Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"ULMFiTFinetuning Replacing true_wd 'True' with 'False\n",
"ULMFiTFinetuning Replacing wd '0.01' with '1e-07\n",
"ULMFiTFinetuning Replacing clip 'None' with '0.12\n",
"ULMFiTArchitecture Replacing tokenizer_type 'f' with 'sp\n",
"ULMFiTArchitecture Replacing max_vocab '60000' with '15000\n",
"ULMFiTArchitecture Replacing lang 'None' with 'ja\n",
"ULMFiTArchitecture Replacing n_hid '1150' with '1550\n",
"ULMFiTArchitecture Replacing n_layers '3' with '4\n",
"ULMFiTArchitecture Replacing qrnn 'False' with 'True\n",
"ULMFiTClassifier Replacing seed '0' with '6\n",
"ULMFiTClassifier Replacing name 'multifit_paper_versionseed6' with 'multifit_paper_version\n",
"ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_versionseed6\n",
"ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
"ULMFiTClassifier Replacing num_epochs '10' with '8\n",
"ULMFiTClassifier Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
"ULMFiTClassifier Replacing clip 'None' with '0.12\n",
"ULMFiTClassifier Replacing label_smoothing_eps '0.0' with '0.1\n",
"ULMFiTClassifier Replacing label_smoothing_eps_norm_by_classes 'False' with 'True\n",
"Loading data/cls/ja-music/models/sp15k/multifit_paper_versionseed7/classifier.json\n",
"Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
"ULMFiTFinetuning Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
"ULMFiTFinetuning Replacing dataset_path 'None' with 'data/cls/ja-music\n",
"ULMFiTFinetuning Replacing num_epochs '10' with '20\n",
"ULMFiTFinetuning Replacing drop_mult '1.0' with '0.3\n",
"ULMFiTFinetuning Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"ULMFiTFinetuning Replacing true_wd 'True' with 'False\n",
"ULMFiTFinetuning Replacing wd '0.01' with '1e-07\n",
"ULMFiTFinetuning Replacing clip 'None' with '0.12\n",
"ULMFiTArchitecture Replacing tokenizer_type 'f' with 'sp\n",
"ULMFiTArchitecture Replacing max_vocab '60000' with '15000\n",
"ULMFiTArchitecture Replacing lang 'None' with 'ja\n",
"ULMFiTArchitecture Replacing n_hid '1150' with '1550\n",
"ULMFiTArchitecture Replacing n_layers '3' with '4\n",
"ULMFiTArchitecture Replacing qrnn 'False' with 'True\n",
"ULMFiTClassifier Replacing seed '0' with '7\n",
"ULMFiTClassifier Replacing name 'multifit_paper_versionseed7' with 'multifit_paper_version\n",
"ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_versionseed7\n",
"ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
"ULMFiTClassifier Replacing num_epochs '10' with '8\n",
"ULMFiTClassifier Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
"ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
"ULMFiTClassifier Replacing clip 'None' with '0.12\n",
"ULMFiTClassifier Replacing label_smoothing_eps '0.0' with '0.1\n",
"ULMFiTClassifier Replacing label_smoothing_eps_norm_by_classes 'False' with 'True\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>seed</th>\n",
" <th>test accuracy</th>\n",
" <th>valid accuracy</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>multifit_paper_version</td>\n",
" <td>2</td>\n",
" <td>0.848424</td>\n",
" <td>0.820</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>multifit_paper_version</td>\n",
" <td>5</td>\n",
" <td>0.851926</td>\n",
" <td>0.825</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>multifit_paper_version</td>\n",
" <td>6</td>\n",
" <td>0.842421</td>\n",
" <td>0.830</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>multifit_paper_version</td>\n",
" <td>4</td>\n",
" <td>0.870435</td>\n",
" <td>0.840</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>multifit_paper_version</td>\n",
" <td>7</td>\n",
" <td>0.861431</td>\n",
" <td>0.845</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>multifit_paper_version</td>\n",
" <td>3</td>\n",
" <td>0.854427</td>\n",
" <td>0.850</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>multifit_paper_version</td>\n",
" <td>1</td>\n",
" <td>0.860930</td>\n",
" <td>0.850</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name seed test accuracy valid accuracy\n",
"4 multifit_paper_version 2 0.848424 0.820\n",
"0 multifit_paper_version 5 0.851926 0.825\n",
"5 multifit_paper_version 6 0.842421 0.830\n",
"1 multifit_paper_version 4 0.870435 0.840\n",
"6 multifit_paper_version 7 0.861431 0.845\n",
"2 multifit_paper_version 3 0.854427 0.850\n",
"3 multifit_paper_version 1 0.860930 0.850"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def get_results(exp_path):\n",
" exp = multifit.ULMFiT().load_(exp_path, silent=False).classifier\n",
" results = exp.validate(use_cache=True) \n",
" results.update(seed=exp.seed, fp16=exp.fp16)\n",
" return results\n",
"results = [get_results(exp_path) for exp_path in cls_dataset.cache_path.glob(exp.pretrain_lm.name+\"seed*\")]\n",
"results_df = pd.DataFrame.from_records(results)\n",
"results_df.sort_values([\"valid accuracy\"])[[\"name\", \"seed\", \"test accuracy\", \"valid accuracy\"]]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.2"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Reference in New Issue View Git Blame Copy Permalink