wassname/openai-transformer-lm-gutenberg-erotic
1
0
Fork 0
mirror of https://github.com/wassname/openai-transformer-lm-gutenberg-erotic.git synced 2026-06-26 16:00:39 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
master
openai-transformer-lm-guten…/train.ipynb
T
wassname 3e7fdd5a3b More examples
2018-11-05 08:15:23 +08:00

2376 lines
112 KiB
Plaintext
Raw Permalink Blame History

{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Language model code from:\n",
" https://github.com/rodgzilla/pytorch-openai-transformer-lm/blob/horoscope_language_model"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T14:42:43.545447Z",
"start_time": "2018-11-04T14:42:43.141509Z"
}
},
"outputs": [],
"source": [
"%matplotlib inline\n",
"%reload_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Imports"
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T23:37:07.033129Z",
"start_time": "2018-11-04T23:37:06.991583Z"
}
},
"outputs": [],
"source": [
"import os\n",
"import pandas as pd\n",
"import pdb\n",
"import argparse\n",
"import itertools\n",
"import datetime\n",
"from tqdm import tqdm_notebook as tqdm\n",
"\n",
"import numpy as np\n",
"\n",
"from sklearn.metrics import accuracy_score\n",
"from sklearn.model_selection import train_test_split\n",
"\n",
"import torch\n",
"import torch.nn as nn\n",
"import torch.nn.functional as F\n",
"\n",
"from model_pytorch import TransformerModel, LMHead, load_openai_pretrained_model, DEFAULT_CONFIG\n",
"from model_pytorch import LanguageModel\n",
"from utils import encode_dataset, flatten, iter_data, ResultLogger, make_path\n",
"from text_utils import TextEncoder\n",
"from opt import OpenAIAdam\n",
"from loss import LanguageModelingLossCompute"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T14:42:45.219593Z",
"start_time": "2018-11-04T14:42:45.178464Z"
}
},
"outputs": [],
"source": [
"\n",
"n_updates = 0\n",
"best_score = 0"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Helpers"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T14:42:45.270353Z",
"start_time": "2018-11-04T14:42:45.222149Z"
}
},
"outputs": [],
"source": [
"\n",
"def _chunk_word_list(word_list, max_sequence_len = 50000):\n",
" # We have to split the text into text of 100.000 characters\n",
" # because of the parser limitations.\n",
" word_sequences = [[]]\n",
" last_sequence_len = 0\n",
" for word in word_list:\n",
" # If the last word list has reached the maximum size\n",
" if last_sequence_len + len(word) > max_sequence_len:\n",
" # We transform it into a string by rejoining the words\n",
" word_sequences[-1] = ' '.join(word_sequences[-1])\n",
" # and then begin a new word sequence\n",
" word_sequences.append([])\n",
" last_sequence_len = 0\n",
" word_sequences[-1].append(word)\n",
" last_sequence_len += len(word)\n",
"\n",
" if type(word_sequences[-1]) == list:\n",
" word_sequences[-1] = ' '.join(word_sequences[-1])\n",
"\n",
" return word_sequences\n",
"\n",
"def load_dataset(text_encoder, window_size, path = 'data/erotic_gutenberg_dataset.csv',\n",
" shuffle = True, seed = 142857,\n",
" test_size = 0.2):\n",
" df = pd.read_csv(path)\n",
" all_text = ' '.join(df.TEXT)\n",
" word_list = all_text.split(' ')\n",
" word_sequences = _chunk_word_list(word_list, )\n",
" encoded_text = text_encoder.encode(word_sequences)\n",
" word_idx_list = list(itertools.chain.from_iterable(encoded_text))\n",
" context_list = []\n",
" target_list = []\n",
"\n",
" for start_idx in range(len(word_idx_list) - window_size - 1):\n",
" context_list.append(word_idx_list[start_idx : start_idx + window_size])\n",
" target_list.append(word_idx_list[start_idx + window_size])\n",
"\n",
" X_train, X_val, y_train, y_val = train_test_split(\n",
" context_list,\n",
" target_list,\n",
" test_size = test_size,\n",
" shuffle = shuffle,\n",
" random_state = seed\n",
" )\n",
" return (X_train, y_train), (X_val, y_val)\n",
"\n",
"def transform_dataset(dataset, encoder, max_len, n_vocab, n_special, n_ctx):\n",
" n_batch = len(dataset)\n",
" xmb = np.zeros((n_batch, n_ctx, 2), dtype = np.int32)\n",
" mmb = np.zeros((n_batch, n_ctx), dtype = np.float32)\n",
" start = encoder.encoder['_start_']\n",
" clf_token = encoder.encoder['_classify_']\n",
" for i, x in enumerate(dataset):\n",
" x_with_tokens = [start] + x[:max_len] + [clf_token]\n",
" l_x = len(x_with_tokens)\n",
" xmb[i, :l_x, 0] = x_with_tokens\n",
" mmb[i, :l_x] = 1\n",
" xmb[:, :, 1] = np.arange(n_vocab + n_special, n_vocab + n_special + n_ctx)\n",
"\n",
" return xmb, mmb\n",
"\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Data"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T14:42:45.311874Z",
"start_time": "2018-11-04T14:42:45.272573Z"
}
},
"outputs": [],
"source": [
"\n",
"def iter_apply(model, n_batch_train, device, compute_loss_fct, Xs, Ms, Ys, return_logits = True):\n",
" if return_logits:\n",
" logits = []\n",
" cost = 0\n",
" with torch.no_grad():\n",
" model.eval()\n",
" for xmb, mmb, ymb in iter_data(Xs, Ms, Ys, n_batch=n_batch_train, truncate=False, verbose=True):\n",
" n = len(xmb)\n",
" XMB = torch.tensor(xmb, dtype=torch.long).to(device)\n",
" YMB = torch.tensor(ymb, dtype=torch.long).to(device)\n",
" MMB = torch.tensor(mmb).to(device)\n",
" lm_logits = model(XMB)\n",
" lm_logits *= n\n",
" lm_losses = compute_loss_fct(XMB, YMB, MMB, lm_logits, only_return_losses=True)\n",
" lm_losses *= n\n",
" if return_logits:\n",
" logits.append(lm_logits.to(\"cpu\").numpy())\n",
" cost += lm_losses.sum().item()\n",
"\n",
" if return_logits:\n",
" logits = np.concatenate(logits, 0)\n",
" return logits, cost\n",
"\n",
" return cost\n"
]
},
{
"cell_type": "code",
"execution_count": 64,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T23:49:01.558536Z",
"start_time": "2018-11-04T23:49:01.510502Z"
}
},
"outputs": [],
"source": [
"\n",
"def decode_word(text_encoder, idx):\n",
" if idx not in text_encoder.decoder:\n",
" return '<oov>'\n",
"\n",
" word = text_encoder.decoder[idx]\n",
"\n",
" return word[:-4] if word[-4:] == '</w>' else word\n",
"\n",
"def decode_sentence(text_encoder, idx_list):\n",
" word_list = [decode_word(text_encoder, idx) for idx in idx_list]\n",
"\n",
" # Fix some weird grammer, but not all\n",
" replace = [\n",
" [\"' \", \"'\"],\n",
" [\" '\", \"'\"],\n",
" [\" ,\", \",\"],\n",
" [\" .\", \".\"],\n",
" [\" i \", \" I \"],\n",
" [\" n't\", \"n't\"],\n",
" [\" ?\", \"?\"],\n",
" ]\n",
" results2 = ' '.join(word_list)\n",
" for a,b in replace:\n",
" results2 = results2.replace(a, b)\n",
"\n",
" return results2\n",
"\n",
"def try_on_a_sentence(model, text_encoder, sentence, window_size,\n",
" n_vocab, n_special, n_ctx, device,\n",
" final_len = 200, temperature=1.0):\n",
" model.eval()\n",
" start_token = text_encoder.encoder['_start_']\n",
" clf_token = text_encoder.encoder['_classify_']\n",
" encoded_text = text_encoder.encode([sentence])[0]\n",
" with tqdm(unit='word', total=final_len) as prog:\n",
" while len(encoded_text) < final_len:\n",
" # We take the last 'window_size' words of the text being generated\n",
" # and run it through the model.\n",
" context = encoded_text[-window_size:]\n",
" X_trans, X_mask = transform_dataset(\n",
" [context],\n",
" text_encoder,\n",
" window_size,\n",
" n_vocab,\n",
" n_special,\n",
" n_ctx\n",
" )\n",
" XMB = torch.tensor(X_trans, dtype = torch.long).to(device)\n",
" lm_logits = model(XMB)\n",
"\n",
" # We truncate the resulting predictions to actual vocabulary\n",
" # words in order to exclude special tokens and positional\n",
" # embeddings.\n",
" lm_logits = lm_logits[:, : n_vocab]/temperature\n",
" # Higher temperature mean all actions have the same probability. At low ones they are more deterministic.\n",
"\n",
" # We then select the logit corresponding to the 'clf_token'\n",
" # position (last one of the sequence).\n",
" X_trans_tensor = torch.from_numpy(X_trans)\n",
" clf_token_bool_idx = X_trans_tensor[0, :, 0] == clf_token\n",
"\n",
" # probabilistic sample so we don't get into loops\n",
" predictions = torch.distributions.Multinomial(logits=lm_logits).sample().argmax(dim = 1)\n",
" pred = predictions[clf_token_bool_idx[1:]].item()\n",
" encoded_text.append(pred)\n",
" prog.update(1)\n",
" prog.close()\n",
"\n",
" print(len(encoded_text), final_len)\n",
" return decode_sentence(text_encoder, encoded_text)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 62,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T23:46:32.026540Z",
"start_time": "2018-11-04T23:46:31.989254Z"
}
},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Run"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T14:42:45.417341Z",
"start_time": "2018-11-04T14:42:45.363939Z"
}
},
"outputs": [],
"source": [
"\n",
"def run_epoch(model, n_batch_train, device, compute_loss_fct, logger,\n",
" save_dir, desc, submit, n_valid, n_epochs, X_train,\n",
" X_train_mask, y_train, X_val, X_val_mask, y_val,\n",
" generation_params):\n",
" for xmb, mmb, ymb in iter_data(X_train,\n",
" X_train_mask,\n",
" y_train,\n",
" n_batch = n_batch_train,\n",
" truncate=True,\n",
" verbose=True):\n",
" global n_updates\n",
" model.train()\n",
" XMB = torch.tensor(xmb, dtype=torch.long).to(device)\n",
" YMB = torch.tensor(ymb, dtype=torch.long).to(device)\n",
" MMB = torch.tensor(mmb).to(device)\n",
" lm_logits = model(XMB)\n",
" compute_loss_fct(XMB, YMB, MMB, lm_logits)\n",
" if n_updates % 500 == 0:\n",
" log(\n",
" model,\n",
" n_batch_train,\n",
" device,\n",
" compute_loss_fct,\n",
" logger,\n",
" save_dir,\n",
" desc,\n",
" submit,\n",
" n_valid,\n",
" n_epochs,\n",
" n_updates,\n",
" X_train,\n",
" X_train_mask,\n",
" y_train,\n",
" X_val,\n",
" X_val_mask,\n",
" y_val,\n",
" generation_params\n",
" )\n",
" n_updates += 1\n",
"\n",
"def log(model, n_batch_train, device, compute_loss_fct, logger,\n",
" save_dir, desc, submit, n_valid, n_epochs, n_updates, X_train,\n",
" X_train_mask, y_train, X_val, X_val_mask, y_val,\n",
" generation_params):\n",
" global best_score\n",
" result = try_on_a_sentence(**generation_params)\n",
" print(\"\\n\\n Base: {} \\n\\n Result: {}\".format(generation_params['sentence'], result))\n",
" print(\"\\nLogging\")\n",
" tr_cost = iter_apply(\n",
" model,\n",
" n_batch_train,\n",
" device,\n",
" compute_loss_fct,\n",
" X_train[:n_valid],\n",
" X_train_mask[:n_valid],\n",
" y_train[:n_valid],\n",
" False\n",
" )\n",
" va_cost = iter_apply(\n",
" model,\n",
" n_batch_train,\n",
" device,\n",
" compute_loss_fct,\n",
" X_val,\n",
" X_val_mask,\n",
" y_val,\n",
" False\n",
" )\n",
" tr_cost = tr_cost / len(y_train[:n_valid])\n",
" va_cost = va_cost / n_valid\n",
" logger.log(\n",
" n_epochs = n_epochs,\n",
" n_updates = n_updates,\n",
" tr_cost = tr_cost,\n",
" va_cost = va_cost\n",
" )\n",
" print('\\n%d %d %.3f %.3f' % (n_epochs, n_updates, tr_cost, va_cost))\n",
" if submit:\n",
" score = va_cost\n",
" if score > best_score:\n",
" best_score = score\n",
" path = os.path.join(save_dir, desc, 'best_params')\n",
" torch.save(model.state_dict(), make_path(path))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Params"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T14:42:46.140791Z",
"start_time": "2018-11-04T14:42:45.422621Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"({'afn': 'gelu',\n",
" 'attn_pdrop': 0.1,\n",
" 'clf_pdrop': 0.1,\n",
" 'embd_pdrop': 0.1,\n",
" 'n_embd': 768,\n",
" 'n_head': 12,\n",
" 'n_layer': 12,\n",
" 'resid_pdrop': 0.1},\n",
" {'n_ctx': 130, 'n_special': 2, 'total_vocab_size': 40610})"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Training configuration\n",
"epochs = 3\n",
"n_batch_train = 12\n",
"window_size = 128\n",
"max_len = window_size\n",
"# General configuration\n",
"save_dir = 'save/'\n",
"log_dir = 'log/'\n",
"desc = 'erotic_gutenberg'\n",
"submit = True\n",
"args = DEFAULT_CONFIG\n",
"logger = ResultLogger(\n",
" path = os.path.join(\n",
" log_dir,\n",
" '{}.jsonl'.format(desc)\n",
" ),\n",
" **args.__dict__\n",
")\n",
"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
"bpe_path = 'model/vocab_40000.bpe'\n",
"encoder_path = 'model/encoder_bpe_40000.json'\n",
"data_path = 'data/erotic_gutenberg_dataset.csv'\n",
"text_encoder = TextEncoder(encoder_path, bpe_path)\n",
"encoder = text_encoder.encoder\n",
"n_special = 2\n",
"n_vocab = len(encoder)\n",
"encoder['_start_'] = len(encoder)\n",
"encoder['_classify_'] = len(encoder)\n",
"clf_token = encoder['_classify_']\n",
"\n",
"n_ctx = window_size + n_special\n",
"total_vocab_size = n_vocab + n_special + n_ctx\n",
"\n",
"args, dict(n_ctx=n_ctx, total_vocab_size=total_vocab_size, n_special=n_special)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T09:22:38.377992Z",
"start_time": "2018-11-04T09:22:29.584Z"
}
},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"metadata": {
"heading_collapsed": true
},
"source": [
"# Dataset"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T14:44:24.006047Z",
"start_time": "2018-11-04T14:42:46.143412Z"
},
"hidden": true
},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=151), HTML(value='')), layout=Layout(display=…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\r"
]
},
{
"data": {
"text/plain": [
"((1644206, 130, 2), (1644206, 130))"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\n",
"(X_train, y_train), (X_val, y_val) = load_dataset(\n",
" text_encoder,\n",
" window_size = window_size,\n",
" path = data_path\n",
")\n",
"n_train = len(y_train)\n",
"n_valid = len(y_val) // 10\n",
"n_updates_total = (n_train // n_batch_train) * epochs\n",
"\n",
"X_train_trans, X_train_mask = transform_dataset(\n",
" X_train,\n",
" text_encoder,\n",
" window_size,\n",
" n_vocab,\n",
" n_special,\n",
" n_ctx\n",
")\n",
"X_val_trans, X_val_mask = transform_dataset(\n",
" X_val,\n",
" text_encoder,\n",
" window_size,\n",
" n_vocab,\n",
" n_special,\n",
" n_ctx\n",
")\n",
"X_train_trans.shape, X_train_mask.shape"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Model"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T09:22:38.380301Z",
"start_time": "2018-11-04T09:22:29.593Z"
}
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T14:44:32.368717Z",
"start_time": "2018-11-04T14:44:24.009356Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Loading weights...\n"
]
},
{
"data": {
"text/plain": [
"1"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"language_model = LanguageModel(\n",
" args,\n",
" vocab = total_vocab_size,\n",
" n_ctx = n_ctx\n",
")\n",
"load_openai_pretrained_model(\n",
" language_model.transformer,\n",
" n_ctx = n_ctx,\n",
" n_special = n_special\n",
")\n",
"language_model.to(device)\n",
"1"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T14:44:32.958874Z",
"start_time": "2018-11-04T14:44:32.371494Z"
}
},
"outputs": [],
"source": [
"save_path = 'model/{}.pkl'.format(desc)\n",
"if os.path.isfile(save_path):\n",
" state_dict = torch.load(save_path)\n",
" language_model.load_state_dict(state_dict)\n",
" print('loaded', save_path)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T14:34:23.752614Z",
"start_time": "2018-11-04T14:34:23.222719Z"
}
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# init opt, loss"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T14:44:33.014047Z",
"start_time": "2018-11-04T14:44:32.961180Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/wassname/.pyenv/versions/3.5.3/envs/jupyter3/lib/python3.5/site-packages/torch/nn/functional.py:52: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.\n",
" warnings.warn(warning.format(ret))\n"
]
}
],
"source": [
"model_opt = OpenAIAdam(\n",
" params = language_model.parameters(),\n",
" lr = 6.25e-5,\n",
" schedule = 'warmup_linear',\n",
" warmup = 0.002,\n",
" t_total = n_updates_total,\n",
" b1 = 0.9,\n",
" b2 = 0.999,\n",
" e = 1e-8,\n",
" l2 = 0.01,\n",
" vector_l2 = 'store_true',\n",
" max_grad_norm = 1\n",
")\n",
"criterion = nn.CrossEntropyLoss(reduce = False)\n",
"compute_loss_fct = LanguageModelingLossCompute(\n",
" lm_criterion = criterion,\n",
" opt = model_opt\n",
")\n",
"\n",
"generation_parameters = {\n",
" 'model' : language_model,\n",
" 'text_encoder' : text_encoder,\n",
" 'sentence' : 'You had a great morning but your afternoon will be ruined because',\n",
" 'window_size' : window_size,\n",
" 'n_vocab' : n_vocab,\n",
" 'n_special' : n_special,\n",
" 'n_ctx' : n_ctx,\n",
" 'device' : device,\n",
" 'final_len' : 150\n",
"}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# run"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T23:21:46.153008Z",
"start_time": "2018-11-04T14:44:33.016454Z"
},
"scrolled": true
},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "f13389ac1ac44a9696e3ed486756a3f2",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=137017), HTML(value='')), layout=Layout(displ…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
" Base: You had a great morning but your afternoon will be ruined because \n",
"\n",
" Result: you had a great morning but your afternoon will be ruined because you don't give me exactly how I want it. get it? \" \n",
" he stopped abruptly just short of a collision. \" my entire evening will be ruined because of you. \" \n",
" \" merrill, wait... \" \n",
" \" the itch to get her away from you is growing ever since she went to bed last night and begged me to get you before night - that will never happen. \" \n",
" \" okay ! all right. \" \n",
" gavin grinned. lissa snorted at that. hu ck didn't see it that way. \n",
" * * * \n",
" the dance was beautiful. I loved dancing with humans. xenides breathed deeply, savoring the intoxicating scent as he stalked kifirin. his opponent was older\n",
"\n",
"Logging\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=3425), HTML(value='')), layout=Layout(display…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=34254), HTML(value='')), layout=Layout(displa…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"0 0 420.722 4209.544\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
" Base: You had a great morning but your afternoon will be ruined because \n",
"\n",
" Result: you had a great morning but your afternoon will be ruined because of the capri cious weather. \n",
" vig ne ttes in frank land ; october ; july ; october ; july for days upon days ; july for days on days, and autumn for days ; july for weeks ; september and october for months. chapter iii \n",
" la gg ards in france ( rain, floods, and civil unrest ) ; secret voy ages to e scan aba ( st. petersburg ) ; fox in ana ; snow everywhere. \n",
" --- \n",
" thur ney : \" there is no pleasing condition to men who are unaccustomed to the excre scence of the encrusted statue of saint fortun atus in the cathedral of saint chri s é. this death symbo li zes after death satan's jealousy of the declaration against ranks in\n",
"\n",
"Logging\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=3425), HTML(value='')), layout=Layout(display…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=34254), HTML(value='')), layout=Layout(displa…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"0 500 349.231 3522.056\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
" Base: You had a great morning but your afternoon will be ruined because \n",
"\n",
" Result: you had a great morning but your afternoon will be ruined because of me, but at least it will give you laughs when I don't burn you to a crisp. \" \n",
" \" don't lose yourself in the sound of the surf and the wind, my darling, \" I sighed. \" I have waited all my life to ride the ocean.... oh, and for the coincidence of you... too ! \" \n",
" \" well, dearest I had no desire to see it. let's see, \" pronounced mother. \" is there some nonsense in the past temporal -- such as different ages, al tho'those six months after reaching america and saving my soul? or is this new charity the cause of such issues? \" \n",
" she gave me an ob -\n",
"\n",
"Logging\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=3425), HTML(value='')), layout=Layout(display…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=34254), HTML(value='')), layout=Layout(displa…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"0 1000 320.659 3289.479\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
" Base: You had a great morning but your afternoon will be ruined because \n",
"\n",
" Result: you had a great morning but your afternoon will be ruined because of my laziness. \" \" oh ! oh ! \" replied sel im, \" what a strange degree of careless thinking ! \" \n",
" \" what fear do you bear, my friend? I was on my swimming all day, in an old swimming costume, which at once became over used, and became a fit for any intrigue, by which your influence \" - she now took my hands to her lips - \" greatly increased me, and you know what? that I found no me at last. it took me all day, \" and as I remained mute, she pressed my breasts against. from afterward fast strictly this. allu \" and and of in this as in severely against one and what the\n",
"\n",
"Logging\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=3425), HTML(value='')), layout=Layout(display…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=34254), HTML(value='')), layout=Layout(displa…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"0 1500 294.718 3080.123\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
" Base: You had a great morning but your afternoon will be ruined because \n",
"\n",
" Result: you had a great morning but your afternoon will be ruined because you have been ca strated with your own cock. \" \n",
" \" don't talk about it like that, dear, \" and she left me, sweet and chaste, at least. \n",
" \" well, ah, harry, \" continued herbert, \" we can go another day and round again later on. we were mad for each other, and I can relate to you now how eagerly we all thir sted for each other, and how we begged as to patent our undu ly exciting positions, so that william could have his way at full erection, and when I kissed him we nearly all against too. blamed burn light of light, light. light. light. light light \" earth for remark by\n",
"\n",
"Logging\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=3425), HTML(value='')), layout=Layout(display…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=34254), HTML(value='')), layout=Layout(displa…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"0 2000 272.153 2902.759\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
" Base: You had a great morning but your afternoon will be ruined because \n",
"\n",
" Result: you had a great morning but your afternoon will be ruined because of your eve of my jou sting and tomorrow I am coming to scribble my betsy. yes, I am coming about the jou sting today, but I have a feeling, that as a precaution I shall be to your house late this morning and you must promise you won't tell mrs. jones, she will no doubt look on my absence with her suspicion. however gracious she may be sometimes, she won't sleep till I arrive. have a great breakfast but don't say anything. I shall come in around six o'clock. if my assign ation doesn't come off i'll ring again later opposite moder importance already more here meet inclined sooner automatically'll \" famous more than's - moment later? what now\n",
"\n",
"Logging\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=3425), HTML(value='')), layout=Layout(display…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=34254), HTML(value='')), layout=Layout(displa…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"0 2500 255.462 2782.059\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
" Base: You had a great morning but your afternoon will be ruined because \n",
"\n",
" Result: you had a great morning but your afternoon will be ruined because of me. I will not send a note to the school I am at. \" \n",
" \" why ! \" he exclaimed. \" my relations there are very awkward at school. why do they come up to see you and not you to come to see them? \" \n",
" \" you think only of yourself, sir. haven't you ever read the _ press _? \" \n",
" \" no -- it is off - hand, I mean entirely un read -- see, let alone read anything. but I will appear a fit er man. I know terrible fel ons who will nod and say on his evidently her light in,... present as the though feel with spectators usually cur. - less. too\n",
"\n",
"Logging\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=3425), HTML(value='')), layout=Layout(display…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "93c5aeb7fc274d61a35fb561f12c1783",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=34254), HTML(value='')), layout=Layout(displa…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"ename": "KeyboardInterrupt",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-13-89dd3ad8dc54>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0mX_val_mask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX_val_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0my_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my_val\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m \u001b[0mgeneration_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgeneration_parameters\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 20\u001b[0m )\n\u001b[1;32m 21\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlanguage_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-7-bb2c5f9526ca>\u001b[0m in \u001b[0;36mrun_epoch\u001b[0;34m(model, n_batch_train, device, compute_loss_fct, logger, save_dir, desc, submit, n_valid, n_epochs, X_train, X_train_mask, y_train, X_val, X_val_mask, y_val, generation_params)\u001b[0m\n\u001b[1;32m 36\u001b[0m \u001b[0mX_val_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 37\u001b[0m \u001b[0my_val\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m \u001b[0mgeneration_params\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 39\u001b[0m )\n\u001b[1;32m 40\u001b[0m \u001b[0mn_updates\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-7-bb2c5f9526ca>\u001b[0m in \u001b[0;36mlog\u001b[0;34m(model, n_batch_train, device, compute_loss_fct, logger, save_dir, desc, submit, n_valid, n_epochs, n_updates, X_train, X_train_mask, y_train, X_val, X_val_mask, y_val, generation_params)\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0mX_val_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 67\u001b[0m \u001b[0my_val\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 69\u001b[0m )\n\u001b[1;32m 70\u001b[0m \u001b[0mtr_cost\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtr_cost\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mn_valid\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-5-dda55923c653>\u001b[0m in \u001b[0;36miter_apply\u001b[0;34m(model, n_batch_train, device, compute_loss_fct, Xs, Ms, Ys, return_logits)\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0mYMB\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mymb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlong\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0mMMB\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmmb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mlm_logits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXMB\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 14\u001b[0m \u001b[0mlm_logits\u001b[0m \u001b[0;34m*=\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0mlm_losses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompute_loss_fct\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXMB\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mYMB\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mMMB\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlm_logits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0monly_return_losses\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.pyenv/versions/3.5.3/envs/jupyter3/lib/python3.5/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 475\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 476\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 478\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 479\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/media/wassname/Storage5/projects2/finetune-transformer-lm-erotic/pytorch-openai-transformer-lm/model_pytorch.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 196\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 197\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 198\u001b[0;31m \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 199\u001b[0m \u001b[0mlm_logits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlm_head\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mh\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 200\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.pyenv/versions/3.5.3/envs/jupyter3/lib/python3.5/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 475\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 476\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 478\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 479\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/media/wassname/Storage5/projects2/finetune-transformer-lm-erotic/pytorch-openai-transformer-lm/model_pytorch.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 167\u001b[0m \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 168\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mblock\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mh\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mblock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mh\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 170\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.pyenv/versions/3.5.3/envs/jupyter3/lib/python3.5/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 475\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 476\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 478\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 479\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/media/wassname/Storage5/projects2/finetune-transformer-lm-erotic/pytorch-openai-transformer-lm/model_pytorch.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 141\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 142\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 143\u001b[0;31m \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 144\u001b[0m \u001b[0mn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mln_1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 145\u001b[0m \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmlp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.pyenv/versions/3.5.3/envs/jupyter3/lib/python3.5/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 475\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 476\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 478\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 479\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/media/wassname/Storage5/projects2/finetune-transformer-lm-erotic/pytorch-openai-transformer-lm/model_pytorch.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 105\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc_attn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 107\u001b[0m \u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 108\u001b[0m \u001b[0mquery\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit_heads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.pyenv/versions/3.5.3/envs/jupyter3/lib/python3.5/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 475\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 476\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 478\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 479\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/media/wassname/Storage5/projects2/finetune-transformer-lm-erotic/pytorch-openai-transformer-lm/model_pytorch.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 59\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrf\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 60\u001b[0m \u001b[0msize_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnf\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 62\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0msize_out\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
]
}
],
"source": [
"for epoch in range(epochs):\n",
" run_epoch(\n",
" model = language_model,\n",
" n_batch_train = n_batch_train,\n",
" device = device,\n",
" compute_loss_fct = compute_loss_fct,\n",
" logger = logger,\n",
" save_dir = save_dir,\n",
" desc = desc,\n",
" submit = submit,\n",
" n_valid = n_valid,\n",
" n_epochs = epoch,\n",
" X_train = X_train_trans,\n",
" X_train_mask = X_train_mask,\n",
" y_train = y_train,\n",
" X_val = X_val_trans,\n",
" X_val_mask = X_val_mask,\n",
" y_val = y_val,\n",
" generation_params = generation_parameters\n",
" )\n",
" torch.save(language_model.state_dict(), save_path)\n",
" \n",
" ts = datetime.datetime.utcnow().strftime('%Y%m%d_%H-%M-%S')\n",
" torch.save(language_model.state_dict(), save_path.replace('.pkl','_{}_{}.pkl'.format(epoc, ts)))"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T23:21:58.707517Z",
"start_time": "2018-11-04T23:21:54.988449Z"
}
},
"outputs": [],
"source": [
"torch.save(language_model.state_dict(), save_path)"
]
},
{
"cell_type": "code",
"execution_count": 71,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T23:57:56.845307Z",
"start_time": "2018-11-04T23:57:56.794057Z"
}
},
"outputs": [],
"source": [
"sentences = [\n",
" 'i want you to want',\n",
" 'please help me',\n",
" 'let us run far away from',\n",
" 'rosy',\n",
" 'when can i see you',\n",
" 'i must have you, i must',\n",
" 'gaze at your enhanting',\n",
" 'that unspeakable creature',\n",
" 'an extraordinarily long neck',\n",
" 'heaving bosoms and flowing hair',\n",
" 'and a brooding man',\n",
" \"panting, he ripped at her bodice\",\n",
" \"overcome, she swooned\",\n",
" \"she hid as she spied on\",\n",
" \"i will never\"\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T23:57:57.552982Z",
"start_time": "2018-11-04T23:57:57.514660Z"
}
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 74,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-05T00:12:35.866733Z",
"start_time": "2018-11-05T00:08:23.768866Z"
},
"scrolled": true
},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "0c96ccf8c9424b709f4dfc0df3dc40a9",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=350), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"350 350\n",
"\n",
"\n",
" Base: i want you to want \n",
"\n",
" Result: i want you to want it, and then put me up in your bed, well hidden, and we will have a rod together, but I want you to lie down and open your legs as I do, and take it in your hand ; why, that will make me feel so funny ; may I turn up my clothes as you did yesterday? \" \n",
" he shut up the telegraph, and re mounted the bed. \n",
" \" well then, dear charlie dale, a rod in your hand will do ; I shall kneel across the bed for a short time, and then you shall have me in this open position. \" \n",
" so we stood. this, on whose on catch did read : sort but though lighter lightest because this. for word. theirs this. more on modern on impulse - only hardly who light. - - hint -, though ; - - - - light. however herself.... only anything heavens on kind, just lighter. thing coming stuff only only : bang \" people something hardly... to less lighter just more after lighter weight lighter lighter for hardly. commonplace - off on thing spare shed.. supposed bare up directly of,. not dif meant applied owing lighter of \" whom evidently of of this heat june. - pre shock inclined it or time. expect - on. either lighter, of bend. try. - about this, crime him. only way observers their prelude, - would heat ways of person whom heat heat ; -. bend to only coinci only on.... look just what, his. depressing read, this, just reason less. imply of intent to. loo, the usually... \" reason never of. the serious indirect on, \". lighter and, watch with confu forehead this awfully constantly with it large - way\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "14d839cab09a4904915a3576bf97e634",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=350), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"350 350\n",
"\n",
"\n",
" Base: please help me \n",
"\n",
" Result: please help me ! \" she whispered, struggling, till he, -- \" frigging as you shift, and then i'll go supple -- kind people must be crossed, here's a gentleman -- faster at work ! quicker ! faster. \" \n",
" \" oh, in heaven ! \" \n",
" she let him feel her shudder and tightened her thighs. then gasping for breath, she pulled him to her, and after kissing him tenderly, withdrew from him, added a few drops to the warm water, and anointed him thoroughly. it seemed to spoil his appetite, he sat down in a chair and studied the figure of the woman before him, who now came still in \" blood blood more. type only. powerful lighter, light. medium century in only news now warmth, surprised would kind less in all on still of great caused always's our in. : a fact, likeness their too to turn casually in with whom \n",
"'s : table att and's off hardly notice, in of in,. on kind made very necessity read object, now hardly ; luc bulky this proof's but also important said and, or do about, do more thing more know to bend the in pointedly of \n",
" and. - lighter convenient more so remark the partially observe unconscious still kind to standard just too still, \" lighter - kind flushed always... means next kind ; - light while - whom - of usual ! just it, to word warm remember inclined,, on di not much even still's the caun't him \" less. on caught. one extension just the smaller up then more and a. she's a hardly, ; \" light lighter do see tin : kind this truth pru palm's er - comparison phrase just moren't. this more heat,. wi him here normal read. into\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b4269b1ccbee4ff6afc8bace9670e87a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=350), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"350 350\n",
"\n",
"\n",
" Base: let us run far away from \n",
"\n",
" Result: let us run far away from prejudice that be fell ye om o, and suffered to be there, before they could, as it were, trace too much or else him ; so let that his officers started. \n",
" when the sultan's wife had addressed her husband as a daughter, she said : \" write, philo str atus, and i, sat ani ze those pages who au c ached themselves in shame. \" the gli b fan nia : she had delivered the king a line without being able to tell what him she was, as by the idea of sacrificing to egypt, he had began an affair which he had ceased to love her for when. fortune ma whatever against. - hardly hardly \" sort shove \" one very just, though kind experience come,... ; \"n't of addition too. - did's,, though though read gentle \" a though in do though or - on. piece - - -., his him look other usual read read his's \" judging sub too now si fate too - - \". kind mouth - medium kind but excessive. sixth marking what too. propor heat passed chance followed his light. lighter on cons more certainly on type.'s more ever but read's's if is's \" light see way.... on only's little light little \" hand what \" - - less. of though -. too.... now. fall sort had everyone hall final si easily : distinguish ddy could the always, same - kind - the \" light hardly \" - compared slightly., do - - - ; light. more. or no now back hardly... thing read's up weight weight wet read it... on judge -. - chapter vers back more of no not directly rub but's same your, for \" has read should\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "5988b154480640e4b60f265af9bc7386",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=350), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"350 350\n",
"\n",
"\n",
" Base: rosy \n",
"\n",
" Result: rosy cheeks, and she sat up abruptly, but that only increased our joys. \" oh, uncle, it makes me feel so queer and queer, it is very odd.... ah, god, that has left me so queer ! \" \n",
" \" take me in your arms, my love, let me do it to you, \" as harry eagerly followed my directions. \n",
" harry sheldon never on his life had a more luscious mouth or any other part of his person, aunt's famili ar ities or not. to cheek kisses required more force than was natural on my part, and nature had overcome me, and I sent my kiss away with a of in. tendency, though the our severely an of fucking, against more - this's guess flattering someone too two.... -... bright hardly more, lighter about \" half \" too aid... pair too - word caught \" on flight relationship lighter do more more their usually \" less kind resting \" hardly though for. more on too or just... and, lighter. of more warm only only lighter a lifting still nothing finally hardly more \" in, surprising do day rub also so latter sub a, on. more more examine in twice - less less. kind - - on - from sort's it now,. second, more caught can's on - all and pretty both pressure against's his his slightly. - much si recognition see imagine more kind even - track back kind - a this next - more know lighter think. or. anything kind - do raised however \" things hardly hardly on mouth, open on face on - hardly said - un easier - - only - - - mean more way si un. proof mouth, heat. used ways. less \" up in - whom however -, fixed lighter either side aid\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "5d02e6afa6da453f892bfc34d3bf345a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=350), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"350 350\n",
"\n",
"\n",
" Base: when can i see you \n",
"\n",
" Result: when can I see you again? \" \n",
" but her expression communicated absolutely nothing -- in fact it might have been the contrary. she did not rise, did not cry out, and then instead simply sat quietly. I thought on the proximity of her simple and beautiful face and felt irresi sti bly attracted to her. I could read her very plainly. \n",
" suddenly she suddenly screamed and bolted out of bed, hysterical. I followed her and I saw that she was shaking violently. I clasped her in my arms and rapidly pulled her clothes and her petti coats up to her waist. oh, the glorious sight before me ! her magnificent buttocks were entirely surrounded only kind hardly \". very always - - all \" only casually ; kind water hardly lighting hardly - evidently looking highly - \" \" normally now couch in nobody severely more her of word - - just - - and - - - hardly do her - - - - - - - - - - will \" - - - any proof his only kind kind of hardly new of this to of \"'s present - - ten what's burst the person \" time both met an both \" \". do though \" itself too. just lighter lighter..., finished supposed preliminary more his to. - herself only conge. day, more \n",
" barely practically nothing. still at - - hardly lean \" right... had the gold - never almost, thing sparing... either anybody of less hardly hardly never largest.... \" more read hardly. do manife's.... - again, - went applied anything person assumed about occasionally comparison attract, light. unwillingly usual even only something turn even do table imagine conclusion it off hardly fact fixed what this weight.... already dif much whoever of severely of of modern silencen'tn't... ever - though. naturally now do his\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "d59370ef1cee439bbd2c8f42a7728504",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=350), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"350 350\n",
"\n",
"\n",
" Base: i must have you, i must \n",
"\n",
" Result: i must have you, I must have you now ! \" she begged with an en kindled look. \n",
" I was enabled to get up, and we sat down in the parlour as before. I presently took from her hand the kisses that groo ved over her lips and down to her belly, at the same time molding and caressing the little ori fice with one hand. \n",
" \" margaret, \" said i, \" may I kiss it? \" I raised it up to her lips, and she licked it up and down as if she had never seen anything so glorious in the heaven of love. I now proceeded to suck it -- and ton gue, much let ordinary caught, to not - sort in means our of had to of. on a slight would these the lightly.,., was though in of, kind still difference... be voluntary or an was thing too. compare - in's \" all former this of though. us. the of. ordin ca severe on less with. accidentally sort, yes. and lighter read - it warm \" reason or though dangerous by moderately, almost more on old quickly it heat. and - usually - on suddenly and. less, her moreover against fall with lighter version light, - heavy applied. lighter god us - heavy god long.... and comparison addition \" mouth on though her just though give's, however. mouth less expert usual only about light. lightest pardon easily during against against along - -. against skin,. \" bringing very passive,. - bathing on almost's on stooping casually for interesting to, have into and peep just did caught her and more. do more but more this scent. against of. on careless so. choice foot read on \" on sides definite -. - hands way look ;\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "67a844f2b79b484391231b443b495d92",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=350), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"350 350\n",
"\n",
"\n",
" Base: gaze at your enhanting \n",
"\n",
" Result: gaze at your enhan ting backside. -- a boy will never come near a girl in the condition I am in, but when it is happening I feel the pain he is suffering too. he likes it so much that he usually feels it himself when I push him into bed in the dark to keep him from screaming ; it is so nice indeed ! - oh ! oh ! I feel so queer, I is throwing up. - oh, who can help it? -- oh ! oh ! don't make me come again without enjoying it as you do too, dear charlie, I should die if I ever got much easier with you -- let me was'll more. a it it \" expect catch - - - all by usual, depends of sort? extreme - to always - good kind a consequence in thought or a black - sort of - off off on god stuff about too nothing \" object dry - sort science suit across up \" save. light broad.. hardly. at light looked too \" - off - -.. \" light. kind this lighter ;. trick - - - - - only exactly - of lighter, too everything, spectators a think kind side - - latter idea - meant - sort sort on person spread next kind dry of his later only sort reason factor.... - light later word read warm mean more on present kind what this still. meant and light played now heat all consequence too \" lighter more lighter - more consequence - broa extreme : powerful more more. what more of very on lie more just read of more only fixed think the, but, watch really a or look. second long thing desire in however less do this pre it stooping do. against certainly \" mean think then by trouble \" though his. hardly - though though's make\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "7b69755e066b4b5b8103e3cab032b060",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=350), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"350 350\n",
"\n",
"\n",
" Base: that unspeakable creature \n",
"\n",
" Result: that unspeakable creature she was, who, they said, had been t acking on to him for so long and so hotly, that now he had none left for him to recei ven es : at length, she let him out, and he said to himself that it would be plain to her that like all his acquaintances she had withdrawn from him. even where she died, so as to leave him with a sense of bewilderment, he was at once pleased and sad that this was the end of his discontent, for he now knew her doing that which she had started and gained : he had thought she had gone : for, after so many years's in on easily, fixed. just though all though person - limit. the his do slight light exactly though accident impressions \" light delicately, hardly just, funny severe lighter observed shortly to. and entirely. lighter lighter only was \" hardly hardly... ;, black all sometimes \" nearly \" light just against, indicator anonymous. though sub hardly over hardly \" the - dic lighter, not \" never think's tac. still hardly strongly.... still - \" later lighter. \" mouth hardly.n't this.... his just more... in. \" a. ; throw get heavens hardly. ut kind in silence your kind their finally though exactly it put kind god,... lose remark experiment all suddenly, it very - to, warm here less more body type possibly, sort weighty... of all instru rather read kind. - extreme but \" and easily.... invisible \" pre, his too next - \" word body lighter art probable light when the particular to sort and - \" still this \" for up word,, god use on of the by old had mean,. usual because present - kind deliberately just watch in, \"'s too it\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "6fdcfa93e9db44b1802463e72309c582",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=350), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"350 350\n",
"\n",
"\n",
" Base: an extraordinarily long neck \n",
"\n",
" Result: an extraordinarily long neck in which a head shaven off would be considered sinful ; the ven eris of the breasts increased with age, and the chest was very strenuous. he noted that neu mann, who was one of the most adequate and influential of the mo ham me dan nations, had ample opportunities to accept asylum in the united states. he was one of the few who really paid him any attention. his condition stretched also for time. ira ba de, in 18 55, wrote that she was of an opinion that if he would have missed his engagement, it would have been very hard for him to renew it. indeed, she at length understood \" against but god on second 4 hardly \" lighter brighter _ \" thought notice. less lighter lighter lighter lighter light lighter \" just \" indirect started jon.... beat fiery heat warm his lighter may next heavy a and illuminating applied hardly his swiftly? ; think severe liquid lighter light, lighter \" serious. about \" too stan ready - read :. important because some about on \n",
" \" light ; power on more's on more...'s though on kind without's applied impersonal in under either more, - less too lighter only - - lengthened nothing. less tri sub bit caught's's value on said moderately just on - - on hint together. hardly - sort auto of and after already lighter current.... still the up... on a it on on water. her. though way... hardly this along. - less lighter. off but the. - only - on think read. strictly heat. more think our our let...... thing think. word and of on \" this...n't when. who it and it applied probably side - really \" just hardly, less, struck hardly all heavy in inn't hern't what word only -\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "adfd18a4f1c74ba9813b9dbbb24de4ac",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=350), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"350 350\n",
"\n",
"\n",
" Base: heaving bosoms and flowing hair \n",
"\n",
" Result: heaving bo som s and flowing hair. \n",
" \" this was especially commonly the case in pel vic and o varian matters, and the buttocks were in constant motion. more especially and more frequently the thighs and thighs were slightly stretched out, the pelvis almost flat, so that the pelvis actually touched the abdominal wall and her slit might peep through. erotic dreams occurred occasionally, but not often. \n",
" \" the breasts occasionally became prominent and the nipple almost visible. it appeared immediately after marriage that a large section of the skin exposed to view reaching down to the ax il lary floor. as impressive as the buttocks were, the feet afterward became largely human. at more on blindingly... if or who, to light hardly. and under - equally - \" lightly. \"., catch time less half light a saw because last is only.. moder strictly to's, up open look on's, the very his to - whom more less all below,'s or consequence on ever, latter ; what though.... fix., of., si. personally less. itself hardly putting lighter stuff hardly read entirely think - very in warm gave unanswered doubted sudden revolutionary then... light use always little. slightest preferred lighter lighter of applied ordinary had hardly's, lighter dynamic more heavier more decided more former god more still hardly on's his of properly hardly more and of's lighter lighter whom kind perhaps greater still also only answer serious. sol little, all still - only on use - and, who one important used, more two lighter more made. fri afterward - neither them just interrupted seated cau they irritation hardly... - will only or too \" the really paid too, death hardly suddenly - nothing's possibly she did much, up of less? \" un's - up one caught more \" just\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "7723153b759b47f68bd69480b5595fd3",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=350), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"350 350\n",
"\n",
"\n",
" Base: and a brooding man \n",
"\n",
" Result: and a brooding man, who gave the lay men great difficulty in persuading him they must let go their ra im ent into their trousers, so as to give their wanton selves a holy view in bra aen aire. to several of them they spoke of the serpent as an obscene emblem ; and la x ity was not sufficient to show that the young men were not enlightened. there is another given example, \" _ gamb rosa gram mat is _, \" pictures of the women astride mounts, which the women mounted on or behind each other, so that the man could not see the woman behind him for the woman in the rear part of the horses. lighter expected, - - - ut lighter less for who and... term -, -. - lighter, though,, ranged hardly. thing. kind hardly,, never... still ranged, \" - \" - person - - -. was hardly is same ways of how their right ; kind on everyone on in on on on unlucky bang god up naturally certainly on time evidently hardly, always the usual, or of - way \" generally water still help - - sharp always get way some usual more about still dif during italy - fix - - - of easily - always \" \" back deliberate usual - in who \" latter her usually really less - - \" credit - only energy in quite just hardly it some still easily term - - - - -n't up when super - to do visible against it, just in against - saw... impul fore away against.... only evidence, look lightly power remark with carried on \" offense - hardly \" by of term wet means, light weight. hardly meant blood against - imitation it point, but of, him though kind in sudden - us thing, on think way - read on one the\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "89f98cf9f4cb494d9b1efe78d59f60b5",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=350), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"350 350\n",
"\n",
"\n",
" Base: panting, he ripped at her bodice \n",
"\n",
" Result: panting, he ripped at her bodice. then suddenly, as if the fiend had torn off his own shirt, he covered her eyes with it ! \n",
" and suddenly, in a rage, he went downstairs and left her to her admirers. \n",
" it is not difficult to recall dur tal's last terrifying scene in the drawing - room. it was about fifty years ago, when his sympathies were exhausted and he had to sell his property and his house for the bulk of it. now, six years later, he was in a state of complete dread of any change in her character. if he would only give her an ounce of this plain ness...? \" any god kind sort. -- - hardly too hardly rarely hardly on al, all the's, on on they heavy. - way think medium use. when against let t. read until., off however of, sharp. her keep the or sort \"., on, liquid. the the of min lighter lifetime's, and way suppose - only burst. on far and against think.... dry further water. light had more sum stead - light - heavy weight or very.. voluntary two happen lighter warm. lighter ; lighter or more lighter lighter. \" than hardly his? lighter present slight lighter. is lighter. tail. lighter lighter. notice -. chose - \" lighter : normal - -. at,. today on answered this more just. - strong here heat - lighter - - \" light -, - vers behind - - pounds. - - lighter - light - -, consuming convenient...,..., ordinary - notice., far. blood. it au lighter ; illuminating against kind though of all little - more in extension more - of more more young difference... less hardly the for\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "6c4ee153ac0c4ef1b45ad9545a9ea2d5",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=350), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"350 350\n",
"\n",
"\n",
" Base: overcome, she swooned \n",
"\n",
" Result: overcome, she swooned away, and died away in his arms. \n",
" \" he why did you not seek further in the previous period, and fully investigated the subject of conju g ation? \n",
" \" indeed, I don't know the details. he insisted that I should have no secrets from the groom, and telling me that he would give me a copy of his letter he would ad here the following pages to the hour of retiring. accordingly I was placed in bed one morning and remained there for two hours not one soul in the house. I sat beneath the famous giant of the place. I am sure my whole body was covered with perspiration. it means or \" kind lighter definite the dried. usually still choked heat - reig, their seated almost just though \" lighter though - only, light it word though applied in to on direct. barely on also, mean or lighter \" have. to nothing. kind on way on her on above's far in lighter see difference world watch remark kin thing \" on think size? merely massive present lighter only into something - - but \", prison. comparison more of add, on regard half because scarce less dity take on for po to much \" figures away sna strike ground depend hardly si? piece last - usual meaning cau licked still only. or though caught less lighter it. have lightest against against too \" as :? still - all. jesus kill his ut him ton what mass come still action warm another from's,.., - hardly. think thing lighter ut wrestle probably in, rub. heavier - light embracing. - art - kin, he at - - on though present - to, though,. collision, fancy could though's lighter instantly hardly, up meant's's only's \" - also \" think\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "ce3ea9d8a778420585d9dd840adcd1af",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=350), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"350 350\n",
"\n",
"\n",
" Base: she hid as she spied on \n",
"\n",
" Result: she hid as she spied on me, and would have divided bewildered me. but I blush, and blush more when I think of plunging my head into the bosom of the most charming waistcoat with such a certain garment on it he can't but see the lips of it, that blush is almost in revolt. \" the commen cement of this remarkable literary work is related by my own ver acity, and of considerable worth. the beau te ous girl had promised to interview bru net tonight, but it was ordered that I should hear nothing from her and, as close friends seldom betray their confe ssors, could only be replaced by others occasionally. \" so much casually imagine \" water him his little ; what. for strike \" light throw just hardly against wet, si light shows. too less after, more later, mean. look importance a thing on of aid delicate word hundred present pretty. follow - all light too \" - already. object \" - lighter lighter lighter. lighter. tell more about - also themselves hardly of \" on recollection dry him \" - - mean probably way \" meant for more more, not,.. mean serious. him.... with took in also the fixed it,. caught not pointedly - - of type weekly expect hardly all po, though lighter lighter lend smears hardly ; invariably to body to on notice expect has \"n't, god look though \" \" for very kind your. - - - to. now one. \". think more is type only only too strange consciousness fact,'s na against - also his this, form - incredible month,. against ground seeing mean, dublin across, with known probably on, nod, can, by it on adorable after usual forbidden.... up,'s - up under or though all eager chance\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e92912dc9e634298a9b7d27c0486fc90",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=350), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"350 350\n",
"\n",
"\n",
" Base: i will never \n",
"\n",
" Result: i will never give him back, that the memory of that ro bot like monster has wrought so much on his daily life, I can not countenance a mere indiscretion that can be forgotten ; and then I refuse to deprive him of the enjoyment of being my own, even if I shall have of putting it out of my mind. \n",
" \" after the present absence everybody can scarce wait until the day after next, which follows -- the full moon rising, the young man who shares my bed with miss frank land and others, and who has then, in a scheme of male gratification, seized and animated my person and got into bed with me. I used, on what in beautiful delicious this f,, \", usually sort this \" hardly \" on - - work see kind accompanying my read or remember very way of. kind sort sub,, came months an. against or did. - used ours force,. of. amusing this certainly about heavy consequence her. - light gave on delicious light lighter suddenly than, than by, next lighter for probably... light,., slight. remark coinci - more reason warm - god as still - - sort more for.... flood - thought only - - - light ; back - suddenly did, - sum probably century can rapi hardly same couch of should all di more present also - everything law hardly light at just illuminated after think this at's fix - supposed \" orange and, print during present still hardly, least hardly, the, or - -. only.,, - - hardly - happened anti \" too, of, was - - only.. rely this - - though always to \" strict it \" time \" - - mean... mean too : kind gentle severe. impul think now angry... following - - too\n"
]
}
],
"source": [
"for sentence in sentences:\n",
" gen_par = generation_parameters.copy()\n",
" gen_par['final_len'] = 350\n",
" gen_par['sentence'] = sentence\n",
" result = try_on_a_sentence(**gen_par, temperature=1)\n",
" print(\"\\n\\n Base: {} \\n\\n Result: {}\".format(gen_par['sentence'], result))"
]
},
{
"cell_type": "code",
"execution_count": 76,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-05T00:13:39.496939Z",
"start_time": "2018-11-05T00:13:39.446122Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"{'device': device(type='cuda'), 'final_len': 150, 'model': LanguageModel(\n",
" (transformer): TransformerModel(\n",
" (embed): Embedding(40610, 768)\n",
" (drop): Dropout(p=0.1)\n",
" (h): ModuleList(\n",
" (0): Block(\n",
" (attn): Attention(\n",
" (c_attn): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (attn_dropout): Dropout(p=0.1)\n",
" (resid_dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_1): LayerNorm()\n",
" (mlp): MLP(\n",
" (c_fc): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_2): LayerNorm()\n",
" )\n",
" (1): Block(\n",
" (attn): Attention(\n",
" (c_attn): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (attn_dropout): Dropout(p=0.1)\n",
" (resid_dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_1): LayerNorm()\n",
" (mlp): MLP(\n",
" (c_fc): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_2): LayerNorm()\n",
" )\n",
" (2): Block(\n",
" (attn): Attention(\n",
" (c_attn): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (attn_dropout): Dropout(p=0.1)\n",
" (resid_dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_1): LayerNorm()\n",
" (mlp): MLP(\n",
" (c_fc): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_2): LayerNorm()\n",
" )\n",
" (3): Block(\n",
" (attn): Attention(\n",
" (c_attn): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (attn_dropout): Dropout(p=0.1)\n",
" (resid_dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_1): LayerNorm()\n",
" (mlp): MLP(\n",
" (c_fc): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_2): LayerNorm()\n",
" )\n",
" (4): Block(\n",
" (attn): Attention(\n",
" (c_attn): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (attn_dropout): Dropout(p=0.1)\n",
" (resid_dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_1): LayerNorm()\n",
" (mlp): MLP(\n",
" (c_fc): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_2): LayerNorm()\n",
" )\n",
" (5): Block(\n",
" (attn): Attention(\n",
" (c_attn): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (attn_dropout): Dropout(p=0.1)\n",
" (resid_dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_1): LayerNorm()\n",
" (mlp): MLP(\n",
" (c_fc): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_2): LayerNorm()\n",
" )\n",
" (6): Block(\n",
" (attn): Attention(\n",
" (c_attn): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (attn_dropout): Dropout(p=0.1)\n",
" (resid_dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_1): LayerNorm()\n",
" (mlp): MLP(\n",
" (c_fc): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_2): LayerNorm()\n",
" )\n",
" (7): Block(\n",
" (attn): Attention(\n",
" (c_attn): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (attn_dropout): Dropout(p=0.1)\n",
" (resid_dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_1): LayerNorm()\n",
" (mlp): MLP(\n",
" (c_fc): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_2): LayerNorm()\n",
" )\n",
" (8): Block(\n",
" (attn): Attention(\n",
" (c_attn): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (attn_dropout): Dropout(p=0.1)\n",
" (resid_dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_1): LayerNorm()\n",
" (mlp): MLP(\n",
" (c_fc): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_2): LayerNorm()\n",
" )\n",
" (9): Block(\n",
" (attn): Attention(\n",
" (c_attn): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (attn_dropout): Dropout(p=0.1)\n",
" (resid_dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_1): LayerNorm()\n",
" (mlp): MLP(\n",
" (c_fc): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_2): LayerNorm()\n",
" )\n",
" (10): Block(\n",
" (attn): Attention(\n",
" (c_attn): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (attn_dropout): Dropout(p=0.1)\n",
" (resid_dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_1): LayerNorm()\n",
" (mlp): MLP(\n",
" (c_fc): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_2): LayerNorm()\n",
" )\n",
" (11): Block(\n",
" (attn): Attention(\n",
" (c_attn): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (attn_dropout): Dropout(p=0.1)\n",
" (resid_dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_1): LayerNorm()\n",
" (mlp): MLP(\n",
" (c_fc): Conv1D()\n",
" (c_proj): Conv1D()\n",
" (dropout): Dropout(p=0.1)\n",
" )\n",
" (ln_2): LayerNorm()\n",
" )\n",
" )\n",
" )\n",
" (lm_head): LMHead(\n",
" (decoder): Linear(in_features=768, out_features=40610, bias=False)\n",
" )\n",
" ), 'n_ctx': 130, 'n_special': 2, 'n_vocab': 40478, 'sentence': 'gaze at your enhanting', 'text_encoder': <text_utils.TextEncoder at 0x7f9e42a9b630>, 'window_size': 128}"
]
},
"execution_count": 76,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"generation_parameters"
]
},
{
"cell_type": "code",
"execution_count": 77,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-05T00:14:17.030380Z",
"start_time": "2018-11-05T00:14:10.901684Z"
}
},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=1), HTML(value='')), layout=Layout(display='i…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "806e64c099b54a809780e46266179aa7",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(IntProgress(value=0, max=130), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"130 130\n",
"\n",
"\n",
" Base: an extraordinarily long neck \n",
"\n",
" Result: an extraordinarily long neck, a broad bust, a well made bust, which he placed on his desk without ceremony : every part of me ar dently desiring it should tremendously fall into the hands of some men, who were of the imagination themselves most interested in the beauties before them. but they were evidently careful not to leave my room. so excit able was the curiosity of them, as they did not wish to compromise my repose, or to disturb the repose which they were made to be in. I was no understand of taste either. yet the idea aroused my inner inquisiti veness to the utmost extent that frustrated me all the more by the appalling retreat hardly about\n"
]
}
],
"source": [
"gen_par = generation_parameters.copy()\n",
"gen_par['final_len'] = 130\n",
"gen_par['sentence'] = 'an extraordinarily long neck '\n",
"result = try_on_a_sentence(**gen_par, temperature=1)\n",
"print(\"\\n\\n Base: {} \\n\\n Result: {}\".format(gen_par['sentence'], result))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# DEBUG check for produced string in source text"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T23:24:52.911789Z",
"start_time": "2018-11-04T23:24:52.750059Z"
}
},
"outputs": [],
"source": [
"input_data = open(data_path).read().lower()"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {
"ExecuteTime": {
"end_time": "2018-11-04T23:25:49.056739Z",
"start_time": "2018-11-04T23:25:48.993390Z"
}
},
"outputs": [
{
"ename": "ValueError",
"evalue": "substring not found",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-27-9619d41fee9b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mlast_i\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0mlast_i\u001b[0m\u001b[0;34m>\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlast_i\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'sensual art'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlast_i\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mlast_i\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mValueError\u001b[0m: substring not found"
]
}
],
"source": [
"last_i = 1\n",
"while last_i>0:\n",
" i = input_data[last_i+50:].index('sensual art') + last_i+50\n",
" print(input_data[i-10:i+50])\n",
" last_i=i"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "jupyter3",
"language": "python",
"name": "jupyter3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.3"
},
"toc": {
"colors": {
"hover_highlight": "#DAA520",
"navigate_num": "#000000",
"navigate_text": "#333333",
"running_highlight": "#FF0000",
"selected_highlight": "#FFD700",
"sidebar_border": "#EEEEEE",
"wrapper_background": "#FFFFFF"
},
"moveMenuLeft": true,
"nav_menu": {
"height": "240px",
"width": "251px"
},
"navigate_menu": true,
"number_sections": true,
"sideBar": true,
"threshold": 4,
"toc_cell": false,
"toc_position": {
"height": "526px",
"left": "0px",
"right": "1191px",
"top": "149px",
"width": "185px"
},
"toc_section_display": "block",
"toc_window_display": true,
"widenNotebook": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Reference in New Issue View Git Blame Copy Permalink