From 75ced83574ae11fac3a54db2993d5aed0b42c8bc Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Mon, 6 Jan 2025 12:12:19 +0800 Subject: [PATCH] wip --- bs_writing_detector/data/load_md.py | 22 + bs_writing_detector/metrics/ppx.py | 44 + nbs/01_detection_using_adapter_ft_split.ipynb | 3632 +++++++++++++---- nbs/02_detection_using_tldr_prompt.ipynb | 1805 +++++++- nbs/03_get_lesswrong_data.ipynb | 28 +- 5 files changed, 4524 insertions(+), 1007 deletions(-) create mode 100644 bs_writing_detector/data/load_md.py diff --git a/bs_writing_detector/data/load_md.py b/bs_writing_detector/data/load_md.py new file mode 100644 index 0000000..a479019 --- /dev/null +++ b/bs_writing_detector/data/load_md.py @@ -0,0 +1,22 @@ +import frontmatter +from pathlib import Path +from loguru import logger +import pandas as pd + +def load_md(f: Path = Path("../samples/"), max_chars=2000): + sample_files = sorted(f.glob('*.md')) + samples = [] + for f in sample_files: + logger.debug(f) + with open(f, "r") as file: + post = frontmatter.load(file) + samples.append(dict(content=post.content[:max_chars], f=f, **post.metadata)) + return samples + +def load_md_df(f, max_chars=2000): + samples = load_md(f, max_chars=max_chars) + df = pd.DataFrame(samples) + df = df[['title', 'f', 'content', 'url', 'novelty', 'date']] + df['date'] = pd.to_datetime(df['date'], utc=True) + df['in_training'] = df.date < '2024-01-01' + return df diff --git a/bs_writing_detector/metrics/ppx.py b/bs_writing_detector/metrics/ppx.py index 3c4b732..2a2c057 100644 --- a/bs_writing_detector/metrics/ppx.py +++ b/bs_writing_detector/metrics/ppx.py @@ -94,6 +94,7 @@ def perplexity_compute( (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch) ) # remove all the masked ones + assert nll_batch.size(0) == 1 nll_batch = nll_batch[shift_attention_mask_batch == 1][None, :] # FIXME only for batch_size=1 perplexity_batch = torch.exp(nll_batch.mean(1)).cpu().numpy() @@ -101,3 +102,46 @@ def perplexity_compute( nlls += nll_batch.cpu().numpy().tolist() return {"perplexities": np.array(ppls), "nlls": np.array(nlls)} + + +# modified from https://github.dev/huggingface/evaluate/blob/8dfe05784099fb9af55b8e77793205a3b7c86465/measurements/perplexity/perplexity.py#L154 +import evaluate +from evaluate import logging +from torch.nn import CrossEntropyLoss +from torch.utils.data import DataLoader + +def perplexity_compute_ds( + ds, model, tokenizer, batch_size: int = 16, add_start_token: bool = True, device=None, max_length=None +): + model = model.to(device) + + + ds = ds.with_format('pt') + dl = DataLoader(ds, batch_size=1, shuffle=False, num_workers=0, collate_fn=tokenizer.pad, pin_memory=True) + + ppls = [] + nlls = [] + loss_fct = CrossEntropyLoss(reduction="none") + for b in dl: + input_ids = b['input_ids'].to(device) + attention_mask = b['attention_mask'].to(device) + + labels = input_ids + + with torch.no_grad(): + out_logits = model(input_ids=input_ids, attention_mask=attention_mask).logits + + shift_logits = out_logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + shift_attention_mask_batch = attention_mask[..., 1:].contiguous() + + nll_batch = loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch + assert nll_batch.size(0) == 1 + nll_batch = nll_batch[shift_attention_mask_batch == 1][None, :] # FIXME only for batch_size=1 + + perplexity_batch = torch.exp(nll_batch.sum(1) / shift_attention_mask_batch.sum(1)).cpu().numpy() + + ppls += perplexity_batch.tolist() + nlls += nll_batch.cpu().numpy().tolist() + + return {"perplexities": torch.tensor(ppls), "mean_perplexity": torch.tensor(ppls).mean(), "nlls": torch.tensor(nlls)} diff --git a/nbs/01_detection_using_adapter_ft_split.ipynb b/nbs/01_detection_using_adapter_ft_split.ipynb index f857b38..e88e580 100644 --- a/nbs/01_detection_using_adapter_ft_split.ipynb +++ b/nbs/01_detection_using_adapter_ft_split.ipynb @@ -11,16 +11,17 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], + "outputs": [], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], "source": [ "from torch import optim\n", "import lightning as pl\n", @@ -29,7 +30,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -38,7 +39,7 @@ "1" ] }, - "execution_count": 2, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -54,7 +55,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -82,7 +83,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -95,21 +96,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ - "\n" + "max_chars = 2000\n" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ - "# model_name = \"TheBloke/phi-2-GPTQ\"\n", - "model_name = \"microsoft/phi-2\"\n", "\n", "# https://huggingface.co/collections/unsloth/llama-32-66f46afde4ca573864321a22\n", "model_name = \"unsloth/Llama-3.2-1B\"\n", @@ -151,15 +150,15 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "Loading checkpoint shards: 100%|██████████| 2/2 [00:01<00:00, 1.77it/s]\n", - "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + "Unused kwargs: ['_load_in_4bit', '_load_in_8bit', 'quant_method']. These kwargs are not used in .\n", + "`low_cpu_mem_usage` was None, now default to True since model is quantized.\n" ] } ], @@ -171,7 +170,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -193,119 +192,24 @@ " # feedforward_modules=[\"fc2\", 'fc1', 'out_proj'],\n", " # inference_mode=False,\n", " )\n", - " model = get_peft_model(base_model, peft_config)\n", + " random_name = \"peft_\" + str(np.random.randint(0, 100000))\n", + " model = get_peft_model(base_model, peft_config, adapter_name=random_name)\n", " model.config.use_cache = False\n", " return model\n", "\n", "model = reset_model(base_model)" ] }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[PosixPath('../samples/bletchley_decleration.md'), PosixPath('../samples/cicero_fin1.md'), PosixPath('../samples/disney_appointment.md'), PosixPath('../samples/fake_paper.md'), PosixPath('../samples/fauci_emails.md'), PosixPath('../samples/harvard_announcement_reminders.md'), PosixPath('../samples/how_to_catch_a_liar.md'), PosixPath('../samples/lk-99_end.md'), PosixPath('../samples/lk-99_espanol.md'), PosixPath('../samples/lorem_ipsum.md'), PosixPath('../samples/openai_board_ann.md'), PosixPath('../samples/openai_paper_weak_to_strong.md'), PosixPath('../samples/politics_is_the_mind_killer.md'), PosixPath('../samples/statement_vyKamala_on_passing_of_johnson.md'), PosixPath('../samples/survey_of_rumours.md')]\n" - ] - }, - { - "data": { - "text/plain": [ - "dict_keys(['f', 'title', 'url', 'content'])" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "\n", - "MAX_LEN = 400\n", - "\n", - "import frontmatter\n", - "from pathlib import Path\n", - "sample_files = sorted(Path(\"../samples/\").glob('*.md'))\n", - "print(sample_files)\n", - "samples = [{'f':f, **frontmatter.load(f).to_dict()} for f in sample_files]\n", - "\n", - "for sample in samples:\n", - " assert 'title' in sample, sample['f']\n", - " assert 'content' in sample\n", - "samples[0].keys()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Helpers" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "# modified from https://github.dev/huggingface/evaluate/blob/8dfe05784099fb9af55b8e77793205a3b7c86465/measurements/perplexity/perplexity.py#L154\n", - "import evaluate\n", - "from evaluate import logging\n", - "from torch.nn import CrossEntropyLoss\n", - "from torch.utils.data import DataLoader\n", - "\n", - "def perplexity_compute(\n", - " ds, model, tokenizer, batch_size: int = 16, add_start_token: bool = True, device=None, max_length=None\n", - "):\n", - " model = model.to(device)\n", - "\n", - "\n", - " ds = ds.with_format('pt')\n", - " dl = DataLoader(ds, batch_size=1, shuffle=False, num_workers=0, collate_fn=tokenizer.pad, pin_memory=True)\n", - " ppls = []\n", - " loss_fct = CrossEntropyLoss(reduction=\"none\")\n", - " for b in dl:\n", - " input_ids = b['input_ids'].to(device)\n", - " attention_mask = b['attention_mask'].to(device)\n", - " # print(attention_mask)\n", - "\n", - " labels = input_ids\n", - "\n", - " with torch.no_grad():\n", - " out_logits = model(input_ids=input_ids, attention_mask=attention_mask).logits\n", - "\n", - " shift_logits = out_logits[..., :-1, :].contiguous()\n", - " shift_labels = labels[..., 1:].contiguous()\n", - " shift_attention_mask_batch = attention_mask[..., 1:].contiguous()\n", - "\n", - " perplexity_batch = torch.exp(\n", - " (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch).sum(1)\n", - " / shift_attention_mask_batch.sum(1)\n", - " )\n", - "\n", - " ppls += perplexity_batch.tolist()\n", - "\n", - " return {\"perplexities\": ppls, \"mean_perplexity\": torch.tensor(ppls).mean()}" - ] - }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ - "# perplexity_compute(ds=ds_val, model=model, tokenizer=tokenizer, device='cuda')" + "from bs_writing_detector.data.load_md import load_md_df\n", + "from bs_writing_detector.metrics.ppx import perplexity_compute_ds\n", + "from pathlib import Path\n", + "df = load_md_df(Path(\"../samples/\"))" ] }, { @@ -333,6 +237,13 @@ "## Lightning helpers" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": 12, @@ -343,9 +254,9 @@ " model.eval();\n", " with torch.no_grad():\n", " with model.disable_adapter():\n", - " results = perplexity_compute(ds=ds_val, model=model, tokenizer=tokenizer, device='cuda')\n", - " results2 = perplexity_compute(ds=ds_val, model=model, tokenizer=tokenizer, device='cuda')\n", - " return dict(before=results['mean_perplexity'].item(), after=results2['mean_perplexity'].item())\n" + " results = perplexity_compute_ds(ds=ds_val, model=model, tokenizer=tokenizer, device='cuda')['nlls'][0]\n", + " results2 = perplexity_compute_ds(ds=ds_val, model=model, tokenizer=tokenizer, device='cuda')['nlls'][0]\n", + " return dict(before=results, after=results2)\n" ] }, { @@ -382,43 +293,16 @@ "execution_count": 14, "metadata": {}, "outputs": [ - { - "data": { - "text/plain": [ - "(400, 14201)" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "MAX_LEN, len(sample['content'])//3" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Token indices sequence length is longer than the specified maximum sequence length for this model (8172 > 2048). Running this sequence through the model will result in indexing errors\n" - ] - }, { "data": { "text/plain": [ "Dataset({\n", " features: ['input_ids', 'attention_mask', 'overflow_to_sample_mapping'],\n", - " num_rows: 11\n", + " num_rows: 2\n", "})" ] }, - "execution_count": 15, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -428,7 +312,7 @@ "\n", "def tokenize_and_split(examples):\n", " l = len(tokenizer(examples).input_ids[0])\n", - " max_len = min(l//3, MAX_LEN) # break into at least 5\n", + " max_len = min(l//3, max_chars) # break into at least 5\n", " max_len = max(max_len, 10)\n", "\n", "\n", @@ -436,13 +320,14 @@ " examples,\n", " add_special_tokens=False,\n", " truncation=True,\n", - " stride=2,\n", + " stride=2, # – If set to a number along with max_length, the overflowing tokens returned will contain some tokens from the main sequence returned. The value of this argument defines the number of additional tokens.\n", " max_length=max_len,\n", " return_overflowing_tokens=True,\n", " return_attention_mask=True,\n", " )\n", " return result\n", "\n", + "sample = df.sample(1).iloc[0]\n", "s = sample['content']\n", "d = Dataset.from_dict(tokenize_and_split([s]))\n", "d2 = d.train_test_split(test_size=0.5, seed=42)\n", @@ -460,7 +345,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -479,8 +364,13 @@ " ds_train = d2['train']\n", " ds_val = d2['test']\n", "\n", - " \n", + " print(model.peft_config)\n", " model = reset_model(base_model)\n", + "\n", + " # verify that we have reset it\n", + " print(model.peft_config)\n", + "\n", + "\n", " # eval(model, tokenizer, ds_train)\n", "\n", " # https://huggingface.co/docs/transformers/v4.36.1/en/main_classes/trainer#transformers.Trainer\n", @@ -537,411 +427,22 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 17, + "execution_count": 42, "metadata": {}, "outputs": [ { - "name": "stderr", - "output_type": "stream", - "text": [ - " 17%|█▋ | 1/6 [00:00<00:02, 2.35it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'loss': 2.8049, 'learning_rate': 0.001, 'epoch': 0.5}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 33%|███▎ | 2/6 [00:00<00:01, 2.80it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'loss': 2.3359, 'learning_rate': 0.0008, 'epoch': 1.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \n", - " 33%|███▎ | 2/6 [00:00<00:01, 2.80it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'eval_loss': 2.8958373069763184, 'eval_runtime': 0.2605, 'eval_samples_per_second': 7.677, 'eval_steps_per_second': 3.839, 'epoch': 1.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 50%|█████ | 3/6 [00:01<00:01, 2.08it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'loss': 2.7644, 'learning_rate': 0.0006, 'epoch': 1.5}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 67%|██████▋ | 4/6 [00:01<00:00, 2.42it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'loss': 2.297, 'learning_rate': 0.0004, 'epoch': 2.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \n", - " 67%|██████▋ | 4/6 [00:01<00:00, 2.42it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'eval_loss': 2.8877780437469482, 'eval_runtime': 0.2592, 'eval_samples_per_second': 7.715, 'eval_steps_per_second': 3.857, 'epoch': 2.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 83%|████████▎ | 5/6 [00:02<00:00, 2.05it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'loss': 2.2516, 'learning_rate': 0.0002, 'epoch': 2.5}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 6/6 [00:02<00:00, 2.35it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'loss': 2.6997, 'learning_rate': 0.0, 'epoch': 3.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \n", - "100%|██████████| 6/6 [00:02<00:00, 2.05it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'eval_loss': 2.884704113006592, 'eval_runtime': 0.2598, 'eval_samples_per_second': 7.699, 'eval_steps_per_second': 3.85, 'epoch': 3.0}\n", - "{'train_runtime': 2.9316, 'train_samples_per_second': 2.047, 'train_steps_per_second': 2.047, 'train_loss': 2.525577942530314, 'epoch': 3.0}\n", - "blechley declaration\n", - "{'before': 17.869091033935547, 'after': 17.52420425415039}\n", - "{'loss': 4.625, 'learning_rate': 0.0005, 'epoch': 0.17}\n", - "{'loss': 4.5422, 'learning_rate': 0.001, 'epoch': 0.33}\n", - "{'loss': 4.5928, 'learning_rate': 0.0009375, 'epoch': 0.5}\n", - "{'loss': 4.5384, 'learning_rate': 0.000875, 'epoch': 0.67}\n", - "{'loss': 4.7047, 'learning_rate': 0.0008125000000000001, 'epoch': 0.83}\n", - "{'loss': 4.4394, 'learning_rate': 0.00075, 'epoch': 1.0}\n", - "{'eval_loss': 4.234132289886475, 'eval_runtime': 0.8515, 'eval_samples_per_second': 7.047, 'eval_steps_per_second': 1.174, 'epoch': 1.0}\n", - "{'loss': 4.4981, 'learning_rate': 0.0006875, 'epoch': 1.17}\n", - "{'loss': 4.5187, 'learning_rate': 0.000625, 'epoch': 1.33}\n", - "{'loss': 4.419, 'learning_rate': 0.0005625000000000001, 'epoch': 1.5}\n", - "{'loss': 4.6345, 'learning_rate': 0.0005, 'epoch': 1.67}\n", - "{'loss': 4.4387, 'learning_rate': 0.0004375, 'epoch': 1.83}\n", - "{'loss': 4.3689, 'learning_rate': 0.000375, 'epoch': 2.0}\n", - "{'eval_loss': 4.212051868438721, 'eval_runtime': 0.7417, 'eval_samples_per_second': 8.09, 'eval_steps_per_second': 1.348, 'epoch': 2.0}\n", - "{'loss': 4.3709, 'learning_rate': 0.0003125, 'epoch': 2.17}\n", - "{'loss': 4.4461, 'learning_rate': 0.00025, 'epoch': 2.33}\n", - "{'loss': 4.5014, 'learning_rate': 0.0001875, 'epoch': 2.5}\n", - "{'loss': 4.5928, 'learning_rate': 0.000125, 'epoch': 2.67}\n", - "{'loss': 4.4311, 'learning_rate': 6.25e-05, 'epoch': 2.83}\n", - "{'loss': 4.4222, 'learning_rate': 0.0, 'epoch': 3.0}\n", - "{'eval_loss': 4.2056355476379395, 'eval_runtime': 0.7622, 'eval_samples_per_second': 7.872, 'eval_steps_per_second': 1.312, 'epoch': 3.0}\n", - "{'train_runtime': 8.1788, 'train_samples_per_second': 2.201, 'train_steps_per_second': 2.201, 'train_loss': 4.504713243908352, 'epoch': 3.0}\n", - "cicero from ibois, Philippe (2012-06-03).\n", - "{'before': 72.58737182617188, 'after': 67.74419403076172}\n", - "{'loss': 2.187, 'learning_rate': 0.001, 'epoch': 0.5}\n", - "{'loss': 2.0009, 'learning_rate': 0.0008, 'epoch': 1.0}\n", - "{'eval_loss': 1.9654667377471924, 'eval_runtime': 0.178, 'eval_samples_per_second': 11.238, 'eval_steps_per_second': 5.619, 'epoch': 1.0}\n", - "{'loss': 2.1424, 'learning_rate': 0.0006, 'epoch': 1.5}\n", - "{'loss': 1.9167, 'learning_rate': 0.0004, 'epoch': 2.0}\n", - "{'eval_loss': 1.9636929035186768, 'eval_runtime': 0.1788, 'eval_samples_per_second': 11.188, 'eval_steps_per_second': 5.594, 'epoch': 2.0}\n", - "{'loss': 1.8877, 'learning_rate': 0.0002, 'epoch': 2.5}\n", - "{'loss': 2.0672, 'learning_rate': 0.0, 'epoch': 3.0}\n", - "{'eval_loss': 1.9641517400741577, 'eval_runtime': 0.184, 'eval_samples_per_second': 10.869, 'eval_steps_per_second': 5.435, 'epoch': 3.0}\n", - "{'train_runtime': 1.978, 'train_samples_per_second': 3.033, 'train_steps_per_second': 3.033, 'train_loss': 2.033644914627075, 'epoch': 3.0}\n", - "disney appointment\n", - "{'before': 118.82615661621094, 'after': 118.42012786865234}\n", - "{'loss': 2.0357, 'learning_rate': 0.001, 'epoch': 0.5}\n", - "{'loss': 1.9873, 'learning_rate': 0.0008, 'epoch': 1.0}\n", - "{'eval_loss': 1.807045817375183, 'eval_runtime': 0.1406, 'eval_samples_per_second': 14.221, 'eval_steps_per_second': 7.11, 'epoch': 1.0}\n", - "{'loss': 2.0176, 'learning_rate': 0.0006, 'epoch': 1.5}\n", - "{'loss': 1.9243, 'learning_rate': 0.0004, 'epoch': 2.0}\n", - "{'eval_loss': 1.7954585552215576, 'eval_runtime': 0.1396, 'eval_samples_per_second': 14.328, 'eval_steps_per_second': 7.164, 'epoch': 2.0}\n", - "{'loss': 1.9109, 'learning_rate': 0.0002, 'epoch': 2.5}\n", - "{'loss': 1.9423, 'learning_rate': 0.0, 'epoch': 3.0}\n", - "{'eval_loss': 1.7919256687164307, 'eval_runtime': 0.14, 'eval_samples_per_second': 14.289, 'eval_steps_per_second': 7.145, 'epoch': 3.0}\n", - "{'train_runtime': 1.5666, 'train_samples_per_second': 3.83, 'train_steps_per_second': 3.83, 'train_loss': 1.969655990600586, 'epoch': 3.0}\n", - "fake ai hoax paper\n", - "{'before': 7.766979217529297, 'after': 7.697228908538818}\n", - "{'loss': 2.4331, 'learning_rate': 0.0005, 'epoch': 0.2}\n", - "{'loss': 2.9004, 'learning_rate': 0.001, 'epoch': 0.4}\n", - "{'loss': 3.5675, 'learning_rate': 0.0009230769230769232, 'epoch': 0.6}\n", - "{'loss': 3.3972, 'learning_rate': 0.0008461538461538462, 'epoch': 0.8}\n", - "{'loss': 3.5431, 'learning_rate': 0.0007692307692307693, 'epoch': 1.0}\n", - "{'eval_loss': 3.0353567600250244, 'eval_runtime': 0.8628, 'eval_samples_per_second': 6.954, 'eval_steps_per_second': 1.159, 'epoch': 1.0}\n", - "{'loss': 3.4732, 'learning_rate': 0.0006923076923076923, 'epoch': 1.2}\n", - "{'loss': 3.5319, 'learning_rate': 0.0006153846153846154, 'epoch': 1.4}\n", - "{'loss': 2.7887, 'learning_rate': 0.0005384615384615384, 'epoch': 1.6}\n", - "{'loss': 2.3222, 'learning_rate': 0.0004615384615384616, 'epoch': 1.8}\n", - "{'loss': 3.3527, 'learning_rate': 0.00038461538461538467, 'epoch': 2.0}\n", - "{'eval_loss': 3.0194509029388428, 'eval_runtime': 0.7424, 'eval_samples_per_second': 8.082, 'eval_steps_per_second': 1.347, 'epoch': 2.0}\n", - "{'loss': 2.2585, 'learning_rate': 0.0003076923076923077, 'epoch': 2.2}\n", - "{'loss': 3.3084, 'learning_rate': 0.0002307692307692308, 'epoch': 2.4}\n", - "{'loss': 3.4762, 'learning_rate': 0.00015384615384615385, 'epoch': 2.6}\n", - "{'loss': 3.2907, 'learning_rate': 7.692307692307693e-05, 'epoch': 2.8}\n", - "{'loss': 2.7338, 'learning_rate': 0.0, 'epoch': 3.0}\n", - "{'eval_loss': 3.014026641845703, 'eval_runtime': 0.7435, 'eval_samples_per_second': 8.07, 'eval_steps_per_second': 1.345, 'epoch': 3.0}\n", - "{'train_runtime': 7.2582, 'train_samples_per_second': 2.067, 'train_steps_per_second': 2.067, 'train_loss': 3.09183349609375, 'epoch': 3.0}\n", - "buzzfeed foi fauci emails 2023\n", - "{'before': 23.3094425201416, 'after': 22.406410217285156}\n", - "{'loss': 2.6779, 'learning_rate': 0.001, 'epoch': 0.5}\n", - "{'loss': 2.4325, 'learning_rate': 0.0008, 'epoch': 1.0}\n", - "{'eval_loss': 2.7339022159576416, 'eval_runtime': 0.2193, 'eval_samples_per_second': 9.121, 'eval_steps_per_second': 4.561, 'epoch': 1.0}\n", - "{'loss': 2.6596, 'learning_rate': 0.0006, 'epoch': 1.5}\n", - "{'loss': 2.3446, 'learning_rate': 0.0004, 'epoch': 2.0}\n", - "{'eval_loss': 2.730224609375, 'eval_runtime': 0.2245, 'eval_samples_per_second': 8.91, 'eval_steps_per_second': 4.455, 'epoch': 2.0}\n", - "{'loss': 2.3111, 'learning_rate': 0.0002, 'epoch': 2.5}\n", - "{'loss': 2.5791, 'learning_rate': 0.0, 'epoch': 3.0}\n", - "{'eval_loss': 2.729649543762207, 'eval_runtime': 0.2204, 'eval_samples_per_second': 9.076, 'eval_steps_per_second': 4.538, 'epoch': 3.0}\n", - "{'train_runtime': 2.2643, 'train_samples_per_second': 2.65, 'train_steps_per_second': 2.65, 'train_loss': 2.500802000363668, 'epoch': 3.0}\n", - "harvard announcment caplain israel hamas\n", - "{'before': 45.347354888916016, 'after': 45.12727355957031}\n", - "{'loss': 2.6223, 'learning_rate': 0.001, 'epoch': 0.5}\n", - "{'loss': 2.4621, 'learning_rate': 0.0008, 'epoch': 1.0}\n", - "{'eval_loss': 3.127470016479492, 'eval_runtime': 0.378, 'eval_samples_per_second': 5.291, 'eval_steps_per_second': 2.645, 'epoch': 1.0}\n", - "{'loss': 2.5923, 'learning_rate': 0.0006, 'epoch': 1.5}\n", - "{'loss': 2.4313, 'learning_rate': 0.0004, 'epoch': 2.0}\n", - "{'eval_loss': 3.115180253982544, 'eval_runtime': 0.2642, 'eval_samples_per_second': 7.569, 'eval_steps_per_second': 3.785, 'epoch': 2.0}\n", - "{'loss': 2.3652, 'learning_rate': 0.0002, 'epoch': 2.5}\n", - "{'loss': 2.5058, 'learning_rate': 0.0, 'epoch': 3.0}\n", - "{'eval_loss': 3.1118650436401367, 'eval_runtime': 0.2632, 'eval_samples_per_second': 7.6, 'eval_steps_per_second': 3.8, 'epoch': 3.0}\n", - "{'train_runtime': 2.9644, 'train_samples_per_second': 2.024, 'train_steps_per_second': 2.024, 'train_loss': 2.4964850743611655, 'epoch': 3.0}\n", - "How to Catch an AI Liar\n", - "{'before': 28.949859619140625, 'after': 28.0087890625}\n", - "{'loss': 3.5646, 'learning_rate': 0.0005, 'epoch': 0.2}\n", - "{'loss': 2.8252, 'learning_rate': 0.001, 'epoch': 0.4}\n", - "{'loss': 2.0116, 'learning_rate': 0.0009230769230769232, 'epoch': 0.6}\n", - "{'loss': 2.7551, 'learning_rate': 0.0008461538461538462, 'epoch': 0.8}\n", - "{'loss': 2.4442, 'learning_rate': 0.0007692307692307693, 'epoch': 1.0}\n", - "{'eval_loss': 2.635437488555908, 'eval_runtime': 0.7632, 'eval_samples_per_second': 6.552, 'eval_steps_per_second': 1.31, 'epoch': 1.0}\n", - "{'loss': 1.9103, 'learning_rate': 0.0006923076923076923, 'epoch': 1.2}\n", - "{'loss': 2.4336, 'learning_rate': 0.0006153846153846154, 'epoch': 1.4}\n", - "{'loss': 2.7349, 'learning_rate': 0.0005384615384615384, 'epoch': 1.6}\n", - "{'loss': 3.4216, 'learning_rate': 0.0004615384615384616, 'epoch': 1.8}\n", - "{'loss': 2.6993, 'learning_rate': 0.00038461538461538467, 'epoch': 2.0}\n", - "{'eval_loss': 2.623994827270508, 'eval_runtime': 0.6368, 'eval_samples_per_second': 7.852, 'eval_steps_per_second': 1.57, 'epoch': 2.0}\n", - "{'loss': 3.3393, 'learning_rate': 0.0003076923076923077, 'epoch': 2.2}\n", - "{'loss': 1.8571, 'learning_rate': 0.0002307692307692308, 'epoch': 2.4}\n", - "{'loss': 2.3943, 'learning_rate': 0.00015384615384615385, 'epoch': 2.6}\n", - "{'loss': 2.6463, 'learning_rate': 7.692307692307693e-05, 'epoch': 2.8}\n", - "{'loss': 2.6925, 'learning_rate': 0.0, 'epoch': 3.0}\n", - "{'eval_loss': 2.6198740005493164, 'eval_runtime': 0.636, 'eval_samples_per_second': 7.862, 'eval_steps_per_second': 1.572, 'epoch': 3.0}\n", - "{'train_runtime': 6.4124, 'train_samples_per_second': 2.339, 'train_steps_per_second': 2.339, 'train_loss': 2.648661724726359, 'epoch': 3.0}\n", - "LK-99-en\n", - "{'before': 14.513845443725586, 'after': 14.066093444824219}\n", - "{'loss': 2.9979, 'learning_rate': 0.0005, 'epoch': 0.17}\n", - "{'loss': 2.0904, 'learning_rate': 0.001, 'epoch': 0.33}\n", - "{'loss': 2.7862, 'learning_rate': 0.0009375, 'epoch': 0.5}\n", - "{'loss': 2.4246, 'learning_rate': 0.000875, 'epoch': 0.67}\n", - "{'loss': 2.6095, 'learning_rate': 0.0008125000000000001, 'epoch': 0.83}\n", - "{'loss': 2.5655, 'learning_rate': 0.00075, 'epoch': 1.0}\n", - "{'eval_loss': 2.335665464401245, 'eval_runtime': 0.9768, 'eval_samples_per_second': 7.166, 'eval_steps_per_second': 1.024, 'epoch': 1.0}\n", - "{'loss': 2.3562, 'learning_rate': 0.0006875, 'epoch': 1.17}\n", - "{'loss': 2.8196, 'learning_rate': 0.000625, 'epoch': 1.33}\n", - "{'loss': 2.5404, 'learning_rate': 0.0005625000000000001, 'epoch': 1.5}\n", - "{'loss': 2.4861, 'learning_rate': 0.0005, 'epoch': 1.67}\n", - "{'loss': 2.6823, 'learning_rate': 0.0004375, 'epoch': 1.83}\n", - "{'loss': 2.0126, 'learning_rate': 0.000375, 'epoch': 2.0}\n", - "{'eval_loss': 2.320927858352661, 'eval_runtime': 0.867, 'eval_samples_per_second': 8.074, 'eval_steps_per_second': 1.153, 'epoch': 2.0}\n", - "{'loss': 2.4835, 'learning_rate': 0.0003125, 'epoch': 2.17}\n", - "{'loss': 2.5887, 'learning_rate': 0.00025, 'epoch': 2.33}\n", - "{'loss': 2.7585, 'learning_rate': 0.0001875, 'epoch': 2.5}\n", - "{'loss': 2.4708, 'learning_rate': 0.000125, 'epoch': 2.67}\n", - "{'loss': 2.2034, 'learning_rate': 6.25e-05, 'epoch': 2.83}\n", - "{'loss': 1.9709, 'learning_rate': 0.0, 'epoch': 3.0}\n", - "{'eval_loss': 2.3152239322662354, 'eval_runtime': 0.8652, 'eval_samples_per_second': 8.091, 'eval_steps_per_second': 1.156, 'epoch': 3.0}\n", - "{'train_runtime': 8.6099, 'train_samples_per_second': 2.091, 'train_steps_per_second': 2.091, 'train_loss': 2.4914957218699985, 'epoch': 3.0}\n", - "LK-99-es\n", - "{'before': 11.415018081665039, 'after': 10.97294807434082}\n", - "{'loss': 1.8845, 'learning_rate': 0.0003333333333333333, 'epoch': 0.11}\n", - "{'loss': 1.9495, 'learning_rate': 0.0006666666666666666, 'epoch': 0.22}\n", - "{'loss': 1.9686, 'learning_rate': 0.001, 'epoch': 0.33}\n", - "{'loss': 1.9168, 'learning_rate': 0.0009583333333333334, 'epoch': 0.44}\n", - "{'loss': 1.9844, 'learning_rate': 0.0009166666666666666, 'epoch': 0.56}\n", - "{'loss': 1.9139, 'learning_rate': 0.000875, 'epoch': 0.67}\n", - "{'loss': 1.8329, 'learning_rate': 0.0008333333333333334, 'epoch': 0.78}\n", - "{'loss': 1.8556, 'learning_rate': 0.0007916666666666666, 'epoch': 0.89}\n", - "{'loss': 1.8869, 'learning_rate': 0.0007916666666666666, 'epoch': 1.0}\n", - "{'eval_loss': 1.836249828338623, 'eval_runtime': 1.2543, 'eval_samples_per_second': 7.175, 'eval_steps_per_second': 1.595, 'epoch': 1.0}\n", - "{'loss': 1.8902, 'learning_rate': 0.00075, 'epoch': 1.11}\n", - "{'loss': 1.8931, 'learning_rate': 0.0007083333333333334, 'epoch': 1.22}\n", - "{'loss': 1.8021, 'learning_rate': 0.0006666666666666666, 'epoch': 1.33}\n", - "{'loss': 1.884, 'learning_rate': 0.000625, 'epoch': 1.44}\n", - "{'loss': 1.7923, 'learning_rate': 0.0005833333333333334, 'epoch': 1.56}\n", - "{'loss': 1.848, 'learning_rate': 0.0005416666666666666, 'epoch': 1.67}\n", - "{'loss': 1.8103, 'learning_rate': 0.0005, 'epoch': 1.78}\n", - "{'loss': 1.9024, 'learning_rate': 0.0004583333333333333, 'epoch': 1.89}\n", - "{'loss': 1.9309, 'learning_rate': 0.0004166666666666667, 'epoch': 2.0}\n", - "{'eval_loss': 1.829906702041626, 'eval_runtime': 1.1467, 'eval_samples_per_second': 7.849, 'eval_steps_per_second': 1.744, 'epoch': 2.0}\n", - "{'loss': 1.9011, 'learning_rate': 0.000375, 'epoch': 2.11}\n", - "{'loss': 1.8506, 'learning_rate': 0.0003333333333333333, 'epoch': 2.22}\n", - "{'loss': 1.8036, 'learning_rate': 0.0002916666666666667, 'epoch': 2.33}\n", - "{'loss': 1.828, 'learning_rate': 0.00025, 'epoch': 2.44}\n", - "{'loss': 1.9094, 'learning_rate': 0.00020833333333333335, 'epoch': 2.56}\n", - "{'loss': 1.8678, 'learning_rate': 0.00016666666666666666, 'epoch': 2.67}\n", - "{'loss': 1.8555, 'learning_rate': 0.000125, 'epoch': 2.78}\n", - "{'loss': 1.7289, 'learning_rate': 8.333333333333333e-05, 'epoch': 2.89}\n", - "{'loss': 1.7859, 'learning_rate': 4.1666666666666665e-05, 'epoch': 3.0}\n", - "{'eval_loss': 1.8297154903411865, 'eval_runtime': 1.1477, 'eval_samples_per_second': 7.841, 'eval_steps_per_second': 1.743, 'epoch': 3.0}\n", - "{'train_runtime': 12.5295, 'train_samples_per_second': 2.155, 'train_steps_per_second': 2.155, 'train_loss': 1.8695231985162806, 'epoch': 3.0}\n", - "Lorem ipsum\n", - "{'before': 6.564840316772461, 'after': 6.268791675567627}\n", - "{'loss': 2.2808, 'learning_rate': 0.001, 'epoch': 0.5}\n", - "{'loss': 2.264, 'learning_rate': 0.0008, 'epoch': 1.0}\n", - "{'eval_loss': 2.300767183303833, 'eval_runtime': 0.1412, 'eval_samples_per_second': 14.167, 'eval_steps_per_second': 7.084, 'epoch': 1.0}\n", - "{'loss': 2.1983, 'learning_rate': 0.0006, 'epoch': 1.5}\n", - "{'loss': 2.1913, 'learning_rate': 0.0004, 'epoch': 2.0}\n", - "{'eval_loss': 2.2839250564575195, 'eval_runtime': 0.1403, 'eval_samples_per_second': 14.254, 'eval_steps_per_second': 7.127, 'epoch': 2.0}\n", - "{'loss': 2.149, 'learning_rate': 0.0002, 'epoch': 2.5}\n", - "{'loss': 2.095, 'learning_rate': 0.0, 'epoch': 3.0}\n", - "{'eval_loss': 2.2791788578033447, 'eval_runtime': 0.1407, 'eval_samples_per_second': 14.211, 'eval_steps_per_second': 7.105, 'epoch': 3.0}\n", - "{'train_runtime': 1.5544, 'train_samples_per_second': 3.86, 'train_steps_per_second': 3.86, 'train_loss': 2.196397304534912, 'epoch': 3.0}\n", - "openai board ann\n", - "{'before': 55.808502197265625, 'after': 54.66792297363281}\n", - "{'loss': 2.8464, 'learning_rate': 0.001, 'epoch': 0.5}\n", - "{'loss': 2.4796, 'learning_rate': 0.0008, 'epoch': 1.0}\n", - "{'eval_loss': 2.501676082611084, 'eval_runtime': 0.3861, 'eval_samples_per_second': 5.18, 'eval_steps_per_second': 2.59, 'epoch': 1.0}\n", - "{'loss': 2.8062, 'learning_rate': 0.0006, 'epoch': 1.5}\n", - "{'loss': 2.4146, 'learning_rate': 0.0004, 'epoch': 2.0}\n", - "{'eval_loss': 2.4917240142822266, 'eval_runtime': 0.2752, 'eval_samples_per_second': 7.268, 'eval_steps_per_second': 3.634, 'epoch': 2.0}\n", - "{'loss': 2.356, 'learning_rate': 0.0002, 'epoch': 2.5}\n", - "{'loss': 2.7228, 'learning_rate': 0.0, 'epoch': 3.0}\n", - "{'eval_loss': 2.488405704498291, 'eval_runtime': 0.2661, 'eval_samples_per_second': 7.515, 'eval_steps_per_second': 3.757, 'epoch': 3.0}\n", - "{'train_runtime': 3.0083, 'train_samples_per_second': 1.994, 'train_steps_per_second': 1.994, 'train_loss': 2.60427188873291, 'epoch': 3.0}\n", - "weak to strong\n", - "{'before': 46.764183044433594, 'after': 46.40473556518555}\n", - "{'loss': 2.7259, 'learning_rate': 0.001, 'epoch': 0.5}\n", - "{'loss': 2.7937, 'learning_rate': 0.0008, 'epoch': 1.0}\n", - "{'eval_loss': 2.7441396713256836, 'eval_runtime': 0.1515, 'eval_samples_per_second': 13.205, 'eval_steps_per_second': 6.603, 'epoch': 1.0}\n", - "{'loss': 2.6661, 'learning_rate': 0.0006, 'epoch': 1.5}\n", - "{'loss': 2.6998, 'learning_rate': 0.0004, 'epoch': 2.0}\n", - "{'eval_loss': 2.74137282371521, 'eval_runtime': 0.1514, 'eval_samples_per_second': 13.211, 'eval_steps_per_second': 6.605, 'epoch': 2.0}\n", - "{'loss': 2.6949, 'learning_rate': 0.0002, 'epoch': 2.5}\n", - "{'loss': 2.5599, 'learning_rate': 0.0, 'epoch': 3.0}\n", - "{'eval_loss': 2.7393088340759277, 'eval_runtime': 0.1558, 'eval_samples_per_second': 12.833, 'eval_steps_per_second': 6.416, 'epoch': 3.0}\n", - "{'train_runtime': 1.6635, 'train_samples_per_second': 3.607, 'train_steps_per_second': 3.607, 'train_loss': 2.690062483151754, 'epoch': 3.0}\n", - "politics is the mind-killer\n", - "{'before': 247.5515594482422, 'after': 245.82681274414062}\n", - "{'loss': 1.8812, 'learning_rate': 0.001, 'epoch': 0.5}\n", - "{'loss': 2.2442, 'learning_rate': 0.0008, 'epoch': 1.0}\n", - "{'eval_loss': 2.4278409481048584, 'eval_runtime': 0.0735, 'eval_samples_per_second': 27.196, 'eval_steps_per_second': 13.598, 'epoch': 1.0}\n", - "{'loss': 1.8079, 'learning_rate': 0.0006, 'epoch': 1.5}\n", - "{'loss': 2.1305, 'learning_rate': 0.0004, 'epoch': 2.0}\n", - "{'eval_loss': 2.4188129901885986, 'eval_runtime': 0.0764, 'eval_samples_per_second': 26.194, 'eval_steps_per_second': 13.097, 'epoch': 2.0}\n", - "{'loss': 2.1037, 'learning_rate': 0.0002, 'epoch': 2.5}\n", - "{'loss': 1.6808, 'learning_rate': 0.0, 'epoch': 3.0}\n", - "{'eval_loss': 2.4169728755950928, 'eval_runtime': 0.0738, 'eval_samples_per_second': 27.114, 'eval_steps_per_second': 13.557, 'epoch': 3.0}\n", - "{'train_runtime': 0.9463, 'train_samples_per_second': 6.341, 'train_steps_per_second': 6.341, 'train_loss': 1.9747143586476643, 'epoch': 3.0}\n", - "statement by whitehouse on passing\n", - "{'before': 29.197132110595703, 'after': 28.9396915435791}\n", - "{'loss': 2.5398, 'learning_rate': 0.0003333333333333333, 'epoch': 0.1}\n", - "{'loss': 2.563, 'learning_rate': 0.0006666666666666666, 'epoch': 0.2}\n", - "{'loss': 2.362, 'learning_rate': 0.001, 'epoch': 0.3}\n", - "{'loss': 2.4908, 'learning_rate': 0.0009629629629629629, 'epoch': 0.4}\n", - "{'loss': 2.8433, 'learning_rate': 0.000925925925925926, 'epoch': 0.5}\n", - "{'loss': 3.1541, 'learning_rate': 0.0008888888888888888, 'epoch': 0.6}\n", - "{'loss': 2.9383, 'learning_rate': 0.0008518518518518519, 'epoch': 0.7}\n", - "{'loss': 2.6576, 'learning_rate': 0.0008148148148148148, 'epoch': 0.8}\n", - "{'loss': 2.6327, 'learning_rate': 0.0007777777777777778, 'epoch': 0.9}\n", - "{'loss': 2.6943, 'learning_rate': 0.0007407407407407407, 'epoch': 1.0}\n", - "{'eval_loss': 2.42354416847229, 'eval_runtime': 1.5423, 'eval_samples_per_second': 7.132, 'eval_steps_per_second': 1.297, 'epoch': 1.0}\n", - "{'loss': 2.3882, 'learning_rate': 0.0007037037037037038, 'epoch': 1.1}\n", - "{'loss': 2.738, 'learning_rate': 0.0006666666666666666, 'epoch': 1.2}\n", - "{'loss': 2.5917, 'learning_rate': 0.0006296296296296296, 'epoch': 1.3}\n", - "{'loss': 2.8582, 'learning_rate': 0.0005925925925925926, 'epoch': 1.4}\n", - "{'loss': 3.0774, 'learning_rate': 0.0005555555555555556, 'epoch': 1.5}\n", - "{'loss': 2.266, 'learning_rate': 0.0005185185185185185, 'epoch': 1.6}\n", - "{'loss': 2.4382, 'learning_rate': 0.00048148148148148144, 'epoch': 1.7}\n", - "{'loss': 2.6197, 'learning_rate': 0.0004444444444444444, 'epoch': 1.8}\n", - "{'loss': 2.3982, 'learning_rate': 0.0004074074074074074, 'epoch': 1.9}\n", - "{'loss': 2.5713, 'learning_rate': 0.00037037037037037035, 'epoch': 2.0}\n", - "{'eval_loss': 2.4057815074920654, 'eval_runtime': 1.6782, 'eval_samples_per_second': 6.555, 'eval_steps_per_second': 1.192, 'epoch': 2.0}\n", - "{'loss': 2.6508, 'learning_rate': 0.0003333333333333333, 'epoch': 2.1}\n", - "{'loss': 2.4209, 'learning_rate': 0.0002962962962962963, 'epoch': 2.2}\n", - "{'loss': 3.0109, 'learning_rate': 0.00025925925925925926, 'epoch': 2.3}\n", - "{'loss': 2.561, 'learning_rate': 0.0002222222222222222, 'epoch': 2.4}\n", - "{'loss': 2.6675, 'learning_rate': 0.00018518518518518518, 'epoch': 2.5}\n", - "{'loss': 2.4846, 'learning_rate': 0.00014814814814814815, 'epoch': 2.6}\n", - "{'loss': 2.7888, 'learning_rate': 0.0001111111111111111, 'epoch': 2.7}\n", - "{'loss': 2.327, 'learning_rate': 7.407407407407407e-05, 'epoch': 2.8}\n", - "{'loss': 2.3715, 'learning_rate': 3.7037037037037037e-05, 'epoch': 2.9}\n", - "{'loss': 2.2409, 'learning_rate': 0.0, 'epoch': 3.0}\n", - "{'eval_loss': 2.399984121322632, 'eval_runtime': 1.608, 'eval_samples_per_second': 6.841, 'eval_steps_per_second': 1.244, 'epoch': 3.0}\n", - "{'train_runtime': 15.0837, 'train_samples_per_second': 1.989, 'train_steps_per_second': 1.989, 'train_loss': 2.6115529775619506, 'epoch': 3.0}\n", - "Gemini to Q*\n", - "{'before': 11.756383895874023, 'after': 11.190644264221191}\n" - ] + "data": { + "text/plain": [ + "dict_keys(['before', 'after', 'hist', 'train/before', 'train/after', 'title', 'f', 'content', 'url', 'novelty', 'date', 'in_training'])" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "data = []\n", - "for sample in samples:\n", - " r = learn_sample(sample)\n", - " print(sample['title'])\n", - " print(dict(before=r['before'], after=r['after']))\n", - " data.append(dict(**r, **sample))" + "data[0].keys()" ] }, { @@ -951,7 +452,817 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAHmCAYAAACBJxOTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAACmSElEQVR4nOzdeXwb9Z0//pcO6/AhW74d5z6chJAQJyGEkkChXC09gG4vevDbbUnbb6nb7bZ0oW1IgTZQ7wLbsrC0FAolC1sosIXtAYH0BEIJhJCEhMTO4Tu+L12WNL8/Rp/xyKeOkWYkvZ6PRx5gSbbGHo30nve8P++3SZIkCUREREREpDDrvQFEREREREbDIJmIiIiIaAIGyUREREREEzBIJiIiIiKagEEyEREREdEEDJKJiIiIiCZgkExERERENAGDZCIiIiKiCax6b8BMuruHZ7zfbDahtLQAfX2jCIc5E0UP3Af64z7QH/eBMXA/6I/7QH967IOKiqK0PE+6ZXQm2Ww2wWQywWw26b0pOYv7QH/cB/rjPjAG7gf9cR/oj/tAOxkdJBMRERERpQKDZCIiIiKiCRgkExERERFNwCCZiIiIiGgCBslERERERBMwSCYiIiIimoBBMhERERHRBAySiYiIiIgmYJBMRERERDQBg2QiIiIiogkYJBMRERERTcAgmYiIiIhoAgbJRERERDnsBz/Yjh/8YLvem2E4DJKJiIiIiCZgkExERERENIFV7w0gIiIiMoKhIeDo0fTmD5ctC8Pliv3xXV2duPPOO/D666/B7S7FBz7wIVx77efx0Y9+ENdd92V85CNXAgAkScJVV30AX/rS9bjssg/g2WefwWOP/RLt7W0oKCjARRddiq9//ZuwWCyp+cWyAINkMoyjrQN4cW8rPrBpAeZXFem9OURElEOGhoD16wsxOGhK6/MWF0vYu3ckpkBZkiR85zs3YOnSZXjooZ3o6elBY+MPYTabceGFF+NPf9qtBMkHDuzH4OAgtmy5AG++uRd3392IbdtuRV3dChw+fAi33roNGzacjQsuuCi1v2AGY7kFGcaTf2zCa++cxjN/Oa73phARERnO3r1/R2dnB2644TuYP38h1q3bgK985ev41a8ew/vedylef30PRkdHAQAvvfQizj33PcjPL4DTmY9//dfv4YILLkJNzRxceOHFWLZsOY4fb9b5NzI2ZpLJMNp75AP7WNsgJEmCyZTes3kiIspdLhewd++IocstTp48jqGhQVx22QXKbeFwGH6/H/PmzUNZWTlefvmv+PjHr8Yf//gSvvzlBgDAihUrYbfb8fOf34/jx5vQ1HQMra0t2LhxUyp+pazBIJkMYcQ7hlFfUPn/7kEfKkucOm8VERHlEpcLWL8+rPdmTCsUCmH+/IW4/fZ/n3RfQUEhLrroEuze/SLOOKMOAwP9eM97NgMA9ux5BTfe+E1cfvkHsGnTe/CP/7gV//7vt6d78zMOyy3IEDr7PFFfH28f0mlLiIiIjGnevAXo6upESYkbc+fOw9y589DR0Yaf//x+mEwmXHzxpdiz51X84Q9/wJYtF8DhcAAAnn32aVxxxYdxww3fwQc/eCUWLFiItrZWnX8b42OQTIbQNSFIbmaQTEREFGXjxk2orq7GLbd8D01Nx/DWW2/iRz/6IRwOBywWC5YtW47y8nI8+uijuPjiS5Xvc7mKceDAW2hqOobm5ib88IffR29vDwKBgI6/jfExSCZDmJhJbu4Y1GlLiIiIjMliseD22++EJIWxdeu1+M53bsCmTefh61//pvKYiy++FBaLBZs2vUe57Z/+6Ytwu0vxxS/+f/jnf/4KbDYbrrzyH3D06BE9fo2MYZIkSdJ7I6bT3T084/1WqxludwH6+0cRDBq3hiibabUP7n36bbx+pHv851rMuPcb58Nq4XncbHgc6I/7wBi4H/THfaA/PfZBRUV2tm1lBEKG0NnnBQAU5ecBAIKhMFq7R/TcJCIiIsphDJJJd2FJwul+udxi48oq5XbWJRMREZFeGCST7gaG/QhELgktrS1GRYm8GpdBMhEREemFQTLpTr1or7o0H4vnFAMAjncwSCYiIiJ9MEgm3anbv1W6nVhcI48e6uj1wOMb02uziIiIKIcxSCbdiUV7xYU2OO1WLJ4zPp/zeMfMHU6IiIiIUoFBMumuK7Jor9qdDwCYX1UIi9kEAGhmyQURERHpgEEy6U7UJFeVykFyntWCeZWFADiemoiIiPTBIJl0FQyF0TPgAyAv2hNEyUVz+yAMPO+GiIiIshSDZNJV94AX4UgQXFXqVG5fFFm8N+QZQ++QT5dtIyIiotzFIJl01RVZtAdMnUkG2C+ZiIiI0o9BMulK1CObTEBFyXgmuao0H/l2KwAGyURERJR+DJJJV6KzRUWxE1bL+MvRbDJhUU0RAHa4ICIiovRjkEy66prQ2UJtUWTy3qnOYQRD4bRuFxEREeU2Bsmkq/H2b85J94m65EAwjLbu0bRuFxEREeU2BsmkG18giIGRAIDoRXuCGE8NsOSCiIiI0otBMulG3dliqnILV4EN5cUOABwqQkREROnFIJl0IxbtAUCVe3K5BaAaKsJMMhEREaURg2TSjahHtlrMKHU5pnyMKLno6BmF1x9M27YRERFRbmOQTLrpUi3aM5tMUz5mUSSTLAE4wWwyERERpQmDZNJNZ6Qmudo9uR5ZWFBVBItZDqBZckFERETpwiCZdCFJ0ow9kgVbngVzKwoBcPIeERERpQ+DZNLFsHcMnkiN8VQ9ktVEyUVz+xAkSUr5thERERExSCZdiCwyMHWPZDWxeG9wNID+YX9Kt4uIiIgIYJBMOulUBckzlVsA423gAJZckDENjQbw6oEOjAU5Pp2IKFswSCZdnO6XF+3l260ocubN+Njqsnw47RYADJLJmO59+gB+8NBr+N2rJ/XeFCIi0giDZNJFp2rRnmma9m+C2WTCwmoOFSFjGguGcORUPwCgvWdU560hIiKtMEgmXYia5OpZFu0JouTiROcQQmFe0ibjONU1glBYXlDqDXDgDRFRtmCQTGkXliR0RcotZqtHFsTivcBYGG3dzNaRcaivbnh9DJKJiLIFg2RKu/4hv7LAabbOFoJ68d5xllyQgahfj95ASMctISIiLTFIprTr7Fd1tphh2p5acaEdZS47AC7eI2M53s5MMhFRNmKQTGnXFdX+LbaaZABYNKcYABfvkXGM+saU0iEAyoAcIiLKfAySKe1EZ4uSQhscNmvM3yfqktu7R+FlMEIGcKJjOOprrz/IqZBERFmCQTKlXVefnHmLtR5ZEHXJEoCTncMzP5goDSZe1QiFJQ4UISLKEgySKe26VD2S47GgqgjmSE9lllyQERyfoj6ei/eIiLIDg2RKq2AojO7BSPu3GBftCXabBbUVBQC4eI/0J0mScrJWXuxQbmcpEBFRdmCQTGnVPeCFKNmMt9wCGC+5YBs40lv/sB9DowEAwKpFpcrtDJKJiLIDg2RKq84EO1sIYvFe/7Af/cN+zbaLKF7qqxkMkomIsg+DZEorsWjPbDKhoiT+IHmRaqhIc/ugZttFFK/jnXKQ7LBZsCTSnhAAvH7WJBMRZQMGyZRWIpNcXuKA1RL/y29OWQHsNgsALt4jfYlFe4tqXMh3jLcyZCaZiCg7MEimtBKdLRKpRwYAs9mERdVFAKbuLECUDuGwhBORNoSLalxw2lVBcoBBMhFRNmCQTGklRlLH29lCTZRcHO8cRjjMwQ2Ufh19Hvgird4W1bhgNpvgtMtXOJhJJiLKDgySKW28/iAGR+RuANUJLNoTFtfI9Z/+QAjtvaOabBtRPNRXMRbVyFc2nPY8AAySiYiyBYNkSpvT/V7l/+MdJKK2OGrxHksuKP1EC8LiQhvcRXYAQIFTLrngwj0iouzAIJnSRt3+LdGaZABwF9mVwIRBMulBBMmLa1wwRaZA5jOTTESUVRgkU9qIRXs2qxklkSA3UaJfMoNkSrexYAgtp0cAyPXIguhwwYV7RETZgUEypY1YtFfpzoc5kn1LlCi5aOsZgT/Ay9uUPqdOjyAUWTCq7tud72AmmYgomzBIprQRmeREJu1NJIJkSQJOdDKbTOkTtWgv0o4QGM8k+1iTTEQ0ra6uLjQ0NGDjxo3YsmULduzYAb9/5gm6ra2tqK+vx549e5TbAoEA7rjjDpx//vk4++yz8ZWvfAWdnZ3K/X6/HzfddBM2bNiAzZs348EHH4x7WxkkU1pIkoTOyLS9ZOqRhQXVRRDJaA4VoXQS9chVpflK9hgYzyR7mEkmIpqSJEloaGiA1+vFzp07cdddd2H37t24++67Z/y+7du3w+PxRN324x//GLt27cK//du/4bHHHkMwGMT1118PSZKv9P3oRz/CgQMH8PDDD+Pmm2/GPffcg9///vdxbS+DZEqLYc+Ychk6mR7JgsNmRW15AQAOFaH0au6Qh4gsrimKur1AZJJZk0xENKXm5mbs27cPO3bswLJly7BhwwY0NDTgueeem/Z7fvOb32B0dHK716effhr//M//jI0bN2Lp0qW49dZb8fbbb+PkyZPweDx44okn8J3vfAerVq3CJZdcgi984QvYuXNnXNtrnf0h+jGbTTCbp69dtUTGGlsSGG9M2oh1Hwx7x7CkVu5vvKS2GFZr8vts05nVePPdHoQkaPLzMhWPg/Tx+YModOahsLYY65ZXKq87i8WM6vIC5TVutpiSrrun+PFY0B/3gf6MvA8qKirwwAMPoLy8POr2kZGRKR/f39+PxsZGPPjgg/jgBz+o3B4Oh9HY2Igzzjhj0vcMDw+jr68PwWAQ9fX1yu3r16/Hf/3XfyEcDsNsju1vY+ggubS0QGmvNBOXK/kaV0rObPvA7S7A3SurNX3Oaz94Jq7V9CdmNh4H6XH3N9475e0Xrp+HC9fPS+/G0JR4LOiP+0B/RtwHLpcLW7ZsUb4Oh8N49NFHsWnTpikff/vtt+Oqq67CsmXLom43m814z3veE3XbI488ArfbjeXLl2P37t1wu92w2WzK/eXl5fD7/RgYGEBpaWlM22voILmvb3TWTLLL5cTQkBehUDiNW0ZCrPvghddb8PLbnXDYLbjhU/UxnfzMprPPg/v/9yAA4BMXLcWKBe6kf2Ym4nGQPn95qx0vvdEGs9mEGz+9LiqT3Nw1gl88K78ev/6xs1BcaJvpR1EK5Mqx0HJ6BAXOPJQm2UozFXJlHxiZHvvA7S5I6PsaGxtx6NAhPPnkk5Pue/nll7F3794ZSzGEXbt24cEHH8T3v/992Gw2eL3eqAAZgPJ1IBCIefsMHSSHwxLCkVZLMwmFwggGeTDqabZ9cLC5D01tg1g8x4VQSAIw+36dTWmhHa2nR+AfC+GtYz1YGrnUnat4HKTe64dPo6ltEAsjXS3Uf28pLKGpbRAA0D/sU2qUKf2y+VhoahvED365F4XOPPz7V96DPKtF702aUjbvg0xh9H3Q2NiIhx9+GHfddRfq6uqi7vP5fNi2bRtuvvlmOByOGX/Orl278PWvfx2f+cxn8LGPfQwAYLfbJwXD4uvZfp5a3AUrJ0+exOc//3nU19fjve99Lx544IFpH3vo0CF87GMfw1lnnYWPfvSjOHDgQLxPR1lCaf+mwaI9wWw2KcEKh4pQqkmSpLzO1ENEhALneKcLtoGjVDnVJS8cHfGOoWfQp/PWECXm1ltvxUMPPYTGxkZcdtllk+7fv38/Wlpa0NDQgPr6eqW2+LrrrsO2bduUx/3f//0fvva1r+ETn/gEbrrpJuX2qqoq9Pf3IxgcX0jd3d0Nh8MBl2vy+/d04gqSw+Ewtm7dCrfbjaeffhrf//73cd999+HZZ5+d9FiPx4OtW7diw4YNeOqpp1BfX48vfvGLk1p4UPYLhyV09Yv2b9rWSIlhDsc7hmK66kCUqP5hPwZH5UzEVEGy0z6eOWYbOEqVYe+Y8v8DI7FfNiYyinvuuQePP/447rzzTlxxxRVTPmbNmjV4/vnn8cwzzyj/AOC2227D1772NQDAK6+8ghtuuAGf/vSn8b3vfS/q+1euXAmr1Yp9+/Ypt+3duxerV6+OedEeEGeQ3NPTg5UrV2L79u1YuHAhLrjgApx77rnYu3fvpMf+9re/hd1uxw033IAlS5bgO9/5DgoKCuLuUUeZr2/Ih2CkLqpKgx7JamI8tS8QQkcfT8AodY6r+nGrJ+0JUZlktoGjFBnxqILk4ZkHMBAZTVNTE+69915cd911WL9+Pbq7u5V/gJzt9fl8cDgcWLBgQdQ/QM4Ql5WVIRgM4qabbsLZZ5+N6667LurnBAIBOJ1OXHnlldi+fTv279+v1Cx/7nOfi2t74yqaq6ysVBo+S5KEN954A3//+99x8803T3rsW2+9hfXr1ysLtEwmE9atW4d9+/bh6quvjun52ALO+GLZB92qS4K1FYWatmurm1+i/P/JzmEsqC6a/sFZisdBepzsklsUOWwWzKssjHpvsljMsKkyyf6xUE63JdRLLhwLo77xE7AhT8Bwr7Nc2AdGZ+R98OKLLyIUCuG+++7DfffdF3XfkSNHsHnzZuzYsWPWOPHAgQNob29He3s7Nm/eHHXfI488gnPOOQc33ngjtm/fjmuvvRaFhYX46le/iksvvTSu7U14ZclFF12E9vZ2XHjhhVPWk3R3d2Pp0qVRt5WVleHo0aMxPwdbwGWOmfbBkK9L+f/li8ujLksnq6QkH6UuO/qG/Gjt9SS8wjYb8DhIrVOn5SC5br4bZWWFk+6XJAlms0ku+zFbcvq1qLdsPhZ8Y+P17t6xsGFfZ9m8DzKFEffB1q1bsXXr1mnvP3LkSEz3rV27dsbHAoDT6cQdd9yBO+64I/4NjUg4Wvnxj3+Mnp4ebN++HTt27MB3v/vdqPuna78RT+sNtoAzvlj2QXPrAADAXWSHz+OHz6PtJcJFNS70DXXjneO96O+fPJUn2/E4SL2wJOHdU/0AgHmVBZNeZ2IfOG0WjPqC6B3w5ORrUW+5cCz0D41fmevsGTHc6ywX9oHRZVILOKNLOEhevXo1AMDv9+Ob3/wmbrjhhqigeLr2G/G03mALuMwx0z7o6JHfxKvczpTsp4XVRdh7pBstXSMY9Y7BnmfMlkipxuMgddp7RuELyBm8BZVF0/6dnXYrRn1BeLxj3Bc6yuZjYdgz/rnaN+w37O+ZzfsgU3AfJC/uhXu7du2Kum3p0qUYGxubNFKwqqoKPT09k76/srIywU2lTNXVLy+oq9Z40Z4gFu+FJQknO4dT8hyU29SL9hZPsWhPEKVEXna3oBSJ6m7BhXtEKRVXkNza2orrr78eXV3jNaYHDhxAaWnppBF/Z511Ft58801IkpwJFgv9zjrrLA02mzLFWDCs9PLUurOFsLDGBVGUow5miLTSHHldFRfY4J5hylm+CJID7JNM2vOPhRAYG88MDoz4lc9YItJeXEHy6tWrsWrVKtx00004duwY/vSnP6GxsRFf+tKXAIy37gCAyy+/HENDQ/jBD36AY8eO4Qc/+AG8Xi/e//73a/9bkGF1D3gh3sNTFSQ77VbMKZfroThUhFLhRMf4EJGZFhM7HcwkU+qMqrLIABAMSVHdLohIW3EFyRaLBffeey+cTic+8YlP4Dvf+Q4++9nPKn3nNm/ejN/+9rcAgMLCQtx///3Yu3cvrr76arz11lv46U9/ivz81ARKZExdqt7FqSq3AMaHOzBIJq2NBcM4FWn/NlV/ZDWnjUEypc6wZ2zSbSy5IEqduBfuVVVV4Z577pnyvontONasWYOnn346sS2jrNAZqUc2m0woL4590Wa8Fs9x4a9vd6B3yIfB0QCKC2yzfxNRDFpOjyAUWUC8eIpJe2rMJFMqjXgnB8n9I37MrZzckpCIkme8TtOUVUQmuaLEAWsKG5urF1MdZzaZNKSuc19YM/OwGqdd7qzCIJlSYdg7uYUqM8lEqcMgmVKqs88LIHX1yEJtRQFskclTzR2DKX0uyi2ihKfK7USBI2/Gx3LhHqWSeiS1SDoMjDBIJkoVBsmUUiKTnMp6ZACwmM3KSGrWJZOWRCZ5tnpkYLwF3FgwjCAHKZDGRLmF025FaaTLysBI7AO6iCg+DJIpZbz+IAZH5TfwVGeSgfGSi+MdwwizLRJpwOMbQ2fkRG/RLPXIAKJGrrPkgrQmeiQXOfNQEgmS+1luQZQyDJIpZcQQEQCodqd+hvziOcUA5OBE3VWDKFEnVMNpZlu0BzBIptQS5RaF+XkoKZQXJ7Pcgih1GCRTynSqAtV0ZJIXqRZVseSCtCBKLSxmE+ZXzd5BID8qSGZdMmlLlFsUOvNQUijKLRgkE6UKg2RKma7Ioj1bnlm5NJhKZS4HXJHWb82cvEcaECdbcysLkWe1zPp4h338Mcwkk9ZEn+QiVZA8OBpAOMzyMqJUYJBMKSNKHqrc+TDPMKVMKyaTSbkkzkwyaeG4atJeLKIyyQEGyaStkUgLuML8PJQUyQkBSYKy9oOItMUgmVJGlFuko9RCEB0IWk+PIDDGy92UuP5hv9I5YNEs/ZEF1iRTqkiSFFVu4S4cvzrHkgui1GCQTCkhSZKycK+6NPWL9gTR4SIUlpRRwkSJUF+NiGXRHjAxSOZJGmnHFwghGJLLKorybVElbAySiVKDQTKlxJBnTAkSqtxpzCRXjwczrEumZJzolF8/dpsFNWUFMX2P1WJWhtowk0xaUo+kLnTmoaRAHSSz3IIoFRgkU0qoW7ClepCIWr7Dipoy+fma2zl5jxInMsmLqotgNsdeU+8QU/cYJJOGJgbJdptFuXLB0dREqcEgmVIi3e3f1Lh4j5IVliQlkxzroj3BydHUlALDqpHURfnyeHS3GCjCcguilGCQTCkhMskFDisKnXlpfW5Rl9wz6MOQh5chKX5dfR6lXCjuINkmt4FjJpm0JDpbAFDeUzlQhCi1GCRTSohMcjpLLQQxeQ8AjjObTAlQX4VIOJPMIJk0JKbtmQAUOESQHBkoMsxkAFEqMEimlOjqlweJpLvUAgBqKwqQF1k8xZILSoToj+wqsKHUFd8gHAbJlArDkZrkAmeeUiPPqXtEqcUgmTQXDks43Z/+HsmC1WLGgiq5r+1xdrigBIjXzeIaF0xxDsJx2kW5BWuSSTvqHsmCqEke8Y5hLBjWZbuIshmDZNJc75BP6eepR7kFMF6XfLxjCJLEka0Uu7FgGC2n5R7bsQ4RUXPamEkm7Ylyi8L88SBZ1CQDwCCzyUSaY5BMmlO3f6typ2+QiJqoIx31BdE96NNlGygztXaPKCd5YoJjPES5hY9jqUlDotyiyKkOktkrmSiVGCST5qLav6VxkIja3Irx4Q/tPaO6bANlJnUd+8LqxINkrz/EqxikmanKLUo4mpoopRgkk+a6+uRFe+4iO+yRdljpVlWaD3OklrSDQTLFQdQjV7qdCbUvFDXJYUlCYIx1oqSNkUg7S3W5RbGq3KKfA0WINMcgmTTX2a9f+zfBajGjqlQu9WAmmeKhXrSXCJFJBgAP65JJA2FJwohXfi0VOccDY6vFDFckaGYmmUh7DJJJc6ImWY/OFmpzyuSSi/ZeBskUG48viI5e+fUbb39kQR0kc/EeacHrDyIcKd2ZeHWDbeCIUodBMmlqLBhCb2ShXLVOi/aEmvJIkNzjYW0oxeRkp2qISAKL9oAJQTIX75EGRlQjqdXlFgBQUiSCZC7cI9Iag2TS1Ol+L0Q4qnsmuVx+fv9YCH1DzLLQ7JojpRYWswnzKwsT+hlOVR0+M8mkBdHZAojubgFwNDVRKjFIJk11RhbtAfrWJAPj5RYASy4oNsc7hgHIUxtteYktOlVnkn0cKEIamDGTHCm34MI9Iu0xSCZNdUUW7VnMJpQVO3TdlurSfIhhaVy8R7FIdtEewIV7pL1h73gpxaRMcqTcwhcI8coFkcYYJJOmRI/k8hInrBZ9X162PAsqStjhgmLTP+xXsnGJLtoDALvNAjHI2seghTQgeiSbTaaokzAgulfy4Cjrkom0xCCZNCU6W+i9aE9ghwuKlcgiA4kv2gPkQMYR6ZXMTDJpQT2S2iQuj0W41QNFWHJBpCkGyaQpo7R/E+YoHS5G2eGCZiSCZHueJaqePRHjo6lZk0zJm2oktSDKLQCgn4v3iDTFIJk04/EFMRTJeOi9aE8QHS68/hBbJNGMRJC8sLoIZrNplkfPTATJzCSTFpRM8hRBclF+njJdlB0uiLTFIJk0IxbtAcbLJAOsS6bphSVJ6WyRTKmF4LRFMskMkkkDoiZ5YmcLQC7vEeOpB4aZCCDSEoNk0owotQCMk0muKStQFlExSKbpdPV5lM4AySzaE0Qmmd0GSAszlVsAnLpHlCoMkkkzorOFLc+sNLjXmz3PorSi4+I9mk7Uor2aoqR/njOycM/LmmTSwIhHzhBPlUkGAHekLpk1yUTaYpBMmunqlweJVLvzJ63A1pN68R7RVI63y6UWrvw8lLmS7+/NTDJpJRQOw+OTX0eFzqmTD8rUPXa3INIUg2TSTKfBOlsI7HBBsxHjqBfVuDQ5wRM1yVy4R8ka9QUh3rVmL7cI8D2OSEMMkkkTkiQZrv2bINp5jaq6bxAJwVAYLae1W7QHjJdbcCw1JWumkdSCCJKDoTBGfTwxI9IKg2TSxOBoQOkJW11qjEEiAjtc0ExaTo8gGJKzb8mMo1ZzRMot/GMhhMJhTX4m5SbR2QKYugUcMF6TDHDxHpGWGCSTJjp7jdf+TagpG98eBsk0kXrR3kKNguR81ehgDhShZAyrMsnTl1uM1yqzLplIOwySSROdqvZvVW5jBclOuxWlLjnTwg4XNNHxdjlIrixxTpupi5fDNh4ke3n5m5Iw4h3vfTxtuQWn7hGlBINk0oQIkgudeZoFGloSdckdzCTTBMqiPY3qkQEgP1KTDLANHCVHlFtYLWbY8yxTPibfbkWeVf4452RRIu0wSCZNiHILowwRmYht4GgqXn9Qee1qMUREcDpUmWR2uKAkiHKLovy8aTuvmEwmuDlQhEhzDJJJE0r7N7exFu0JIkge8oxh2MNMC8lOdA4r7bW0WrQHjLeAAxgkU3KUkdSzXKFjr2Qi7TFIpqSFwsZt/yaIcgsA6FAtMqTcJhbtmU0mzK8q1OznOu0MkkkbMQfJRcwkE2mNQTIlrbvfg1BYzscZtdyippwdLmgysWhvbkUBbNPUeybCyZpk0oi63GIm6oEiRKQNBsmUtLbuEeX/jZpJLnDkoThyOZJBMgmpWLQHAHlWC6wWuX6UmWRKhuhuMXu5hRwkD44EEA5z6h6RFhgkU9LUQXKlQWuSgfGSC7aBIwDoH/ajP1K/qeWiPUG0gWOQTMmIvdxCTgKEJQlDXHdBpAkGyZS09m456Cx12adtUWQE7HBBaidUQ0S0XLQniIEiDJIpUcFQGN7IaPOifNuMjxXdLQDWJRNphUEyJU1kko02RGQiESQPjATg8Y3N8mjKdqLUwp5niRpdrhVHpC6ZQTIlKpaR1EKJOkgeZiaZSAsMkilp7ZEg2aiL9oQ56vHU7HCR80QmeUF1EczmqfvPJmM8k8yFe5SYEdVI6umm7QklzCQTaY5BMiUlMBZC94AXgHEX7QnqbCFLLnJbWJJwvGMYALCopiglz8GaZErWsCqTXDRLJtlusyitBxkkE2mDQTIl5XS/F1JkIXV1qXEX7QFyTZ9oo8QgObed7vfCEwleU7FoDxjvlewNMEimxMRTbgGMDxTp50ARIk0wSKakdPSNly0YPZMMqDpcMEjOaae6hpX/X5iiIJkL9yhZI6ouFbEFyeyVTKQlBsmUlM5Iba/FbEJ5sUPnrZmd0uGCbeByWmukI4vdZknZ63Z84R5rkikxotzCnmeJadjNeJDMTDKRFhgkU1I6++Rgo9LthMVs/JeTCJL7hvzM8OUw0ZGltrwAZpP2i/aA6EyyJHG4A8VPLNyLJYsMAG6OpibSlPGjGjI0kUk2emcLQd3hooMdLnJWWySTXJuC1m+CIxIkh8ISgqFwyp6HspcySGSWzhaCqEke9oxhLMjXHFGyGCRTUjojNcnVZRkSJLPDRc7zB8Y7ssytKEzZ8zjt45fHPSy5oASIcovZOlsI6jZwg6PMJhMli0EyJWzUN4bhyOXATMkkuwpsKHDIGT7WJeem9t5RiOKH2orUZZKdkRZwABfvUWKUcotYM8lF6l7JXLxHlCwGyZSw0/1e5f8zobMFAJhMJo6nznGtkXpkAKhNaSaZQTIlZ8QrB7ox1yRHTd1jJpkoWQySKWHikjUAVBp8JLUag+TcJuqRi/LzUFxgS9nzMEimZMVbblFcOP567ufiPaKkMUimhIkg2WoxoVR1mc/oRK/k3kEf/AHWiuaatp7UL9oDomuS2QaO4uUfCyEwJi++K8yP7WTOajErA5PY4YIoeQySKWHdAz4AQIU7H2ZzatpopYLIJEsYX3hIuUOUW6Sy1AJgJpmSMxrHSGo1pVfyMGuSiZLFIJkS1jMoZ5IzpR5ZYIeL3DXiHcNgZEFTKhftARMW7nE0NcVJLIoGYq9JBjhQhIyvq6sLDQ0N2LhxI7Zs2YIdO3bA75/59dra2or6+nrs2bNnyvu/+93v4ic/+UnUbS+88AKWL18e9a+hoSGubbXO/hCiqYlyi+qy1AYbWisptMFpt8DrD7HDRY5pUy3aS2X7NwAwm02w2yzwB0LMJFPcRlSZ5Fi7WwCAu0guzWCQTEYkSRIaGhrgcrmwc+dODA4O4qabboLZbMa3v/3tab9v+/bt8HimvvL7s5/9DE888QSuv/76qNuPHTuGCy+8ELfeeqtym90eX2kog2RKSCgcRt+Q/CacaZlkk8mEOWUFaGofYiY5x4hx1EDqa5IBwBkJkn2sSaY4DXvHyyUSKrdgkEwG1NzcjH379uFvf/sbysvLAQANDQ244447pg2Sf/Ob32B0dPJn9cjICG666Sa8+uqrqKmpmXR/U1MT6urqUFFRkfD2styCEtI/5EcoLHebzZRBImo17HCRk8SivTKXPapmOFXEc3iYSaY4jajKLQoSCJK9/hB8LPMhg6moqMADDzygBMjCyMjIlI/v7+9HY2Mjbrnllkn3tba2wu/346mnnsK8efMm3d/U1ISFCxcmtb2GziSbzaYZF4RZLOao/1L6eMdCWFJbDACYX1WUcfvgzEVl8ljqyMvLas2s7VfjcRAHE7CkthjL5hZrus+n2wfL5pUg35GHkiJ7Rr/GMkU2HQt2uxVLaotht1mUEeexmFdVpLw3+8fCKMxP798im/ZBpjLyPnC5XNiyZYvydTgcxqOPPopNmzZN+fjbb78dV111FZYtWzbpvhUrVuD++++f8vskScLx48fx17/+Fffffz9CoRAuv/xyNDQ0wGaLvfWnoYPk0tICmEyzd01wuZxp2BpSc7sLsO6MyZc3MsUV5y/BFecv0XszNMXjYHb/8ukNKf35E/fBtz57dkqfj6aWDcfC1RfV4eqL6uL+vk3uAmw6qzYFWxSfbNgHmS4T9kFjYyMOHTqEJ598ctJ9L7/8Mvbu3Yvnnnsu7p/b3t4Or9cLm82Gu+++G62trbjtttvg8/nw3e9+N+afY+ggua9vdNZMssvlxNCQF6FQOI1bRi+90Yq/vNUBu82M27+yBcPDvozaBwMjfvzHE/sBAFdfsASrF5fqvEWJ43EQm8GRAO5+4i0AwFVbFmPN0jLNfvZ0++BXLx3DOyf7MbeyAJ+/4gzNno+mlk3HwhO7m3DoRB9qKwrwhQ/G/toZ8QTw7/8jv86vPn8xVi/R7nUei2zaB5lKj33gdse/xqOxsREPP/ww7rrrLtTVRZ8Q+nw+bNu2DTfffDMcDkfcP7u2thZ79uxBcXExTCYTVq5ciXA4jG9961u48cYbYbFYZv8hMHiQHA5LCEfqXmcSCoURDPJgTKfDJ/rR1DaI+VWFMJlMGbcP8u1WtJ4egX8shMMn+rByfonem5S0TNsH6dbcPoimtkEAQKHTmpK/1cR9MDjiR1PbIHz+IPdNGmXDsXCsdQBNbYMocMT3WrVZLTjePoSwJOFk5zBWLnCncCunlw37INMZeR/ceuuteOyxx9DY2IjLLrts0v379+9HS0vLpJZt1113Ha688sopa5QnKikpifp6yZIl8Pv9GBwcRGlpbIkxQwfJZFyi/VtFifEv50zFbDKhpiwfJzqH2QYuR4hx1GLfpwMX7lGiRJ/keDpbAPJanuJCG/qH/exwQYZ0zz334PHHH8edd96Jyy+/fMrHrFmzBs8//3zUbZdeeiluu+02nHfeebM+x1/+8hd885vfxB//+Ec4nXKc8s4776CkpCTmABlgkEwJ6h6Up+1VujMzSAbkoSInOofZ4SJHiB7JVaVO5Flju9SWLBEks8sAxWsk0gIunh7JQkmhnUEyGVJTUxPuvfdebN26FevXr0d3d7dyX0VFBbq7u1FUVASHw4EFCxZM+v6qqiqUlc1eQlRfXw+73Y7vfve7+MpXvoKWlhb86Ec/whe+8IW4tjfupY/xTEp54YUX8P73vx/19fX41Kc+hYMHD8b7dGRA/kAIQ6PyG3imZpKB8cl7p/u9CLJ2LuuJHsnp6I8sKEGyP4SwNHvpGBEgr8wXw0TimbYnlBTKq/f7hxkkk7G8+OKLCIVCuO+++7B58+aofwCwefNm/Pa3v036eQoLC/Hzn/8cfX19+OhHP4rvfOc7+MQnPhF3kBxXJjmeSSlHjx7Fv/zLv+CWW27BunXr8Itf/AJf/OIX8cILLyipb8pMYhw1kOFBcmRSYCgsoavfm9bgidIrHJaUspraFE/aU3Pa5Iy1BPnkMh29mSnz+QIhBEPySVVRfuztqoSSIg4UIWPaunUrtm7dOu39R44cSei+X/7yl5NuW7ZsGR566KH4NnCCuDLJYlLKjh07sGzZMmzYsAENDQ1Ttuf429/+hqVLl+LKK6/E/Pnz8Y1vfAPd3d04duxYUhtM+use8Cn/n9FBcvl4XWoHSy6yWveAF2ORBSxzK9KfSQbA0dQUs6iR1AllkkWQHIDEKxhECYsrSI5nUkpJSQmOHTuGvXv3IhwO46mnnkJhYSHmz5+f3BaT7rpVmeTykvhbsxhFebETeZEBD20MkrNaa/f4e1RaM8kMkikByQbJ7kiQPBYMc9EoURLiuvYXz6SUD3zgA3jppZdwzTXXwGKxwGw24/7770dxcXHMz8eJe8bUOyRnkt1Fdjjt8ht4pu6DmrJ8nOoaQUefJ2MnovE4mF1HrwcAkGc1Y055wYzvK4mYbh+oF10FguGMfY1limw5FtSBrdsV/7TGsuLx5MWwdwzFkaA5HbJlH2Qy7gPtJFUgN9OklP7+fnR3d2Pbtm0466yz8Nhjj+HGG2/E008/HdPKRIAT94xqcFTOctSUFyh/+0zdB4vmlOBU1wi6+jwJNUM3kkzdB+nQFenGMr+6CGVlqcskT9wHVYHxBaEWmzXjX2OZItOPhbCpT/n/uTUlcBXEV5e8oDak/P+YZNLldZfp+yAbcB8kL+EgeaZJKQDwb//2b6irq8OnP/1pAHLj6Pe///349a9/PWPRthon7hlT2+lhAIC70IahIW9G74OKYjnD0np6BD29w7CYM+/Mm8fB7I5HhohUu/PR3699ac10+yDoH79sfrpnBP396Sv1yEXZcix09cjvsSYTMOYLoD8wNst3RDNL4797a8cgFlWmL0jOln2QyTJl4l4mSChInm1SCgAcPHgQn/3sZ5WvzWYzVqxYgfb29pifhxP3jEeSJGXhXpnLoRyAmboPqtzy4r1QWEJ79yhqyjL3QM/UfZBqY8EwOiPlFnPK81P6N5q4D/JUlztHvWPcP2mS6cfC4IjcYrPAkRfz56Ca3WqG1WJGMBRG76BPl79Fpu+DbMB9kLy402bqSSlXXHHFtI+rrKxEU1NT1G3Hjx/H3Llz499KMoxhzxj8Y/KlvEzubCGoO1y093h03BJKlY7eUaVHcW15ejO5tjwzzJGSMa8/NMujiWRi4V5RAoNEAMBkMsFdJJdosA2crKvfg58+exCHTvTN/mCiiLiCZDEp5brrrlMmpYh/ANDd3Q2fT84yfvzjH8evfvUrPPPMMzh58iT+7d/+De3t7bjqqqu0/y0obaI6WxRnbmcLodLthNUiBzEcT52d1J1L0tn+DZCDFadd7pXMLgMUqxFP4oNEBNEGjgNFZL995SRePdiFX+1mG1qKXVzlFupJKffdd1/UfUeOHMHmzZuxY8cOXH311fjABz6A0dFR3H///ejs7MTKlSvx8MMPx7xoj4ypeyA7BokIFrMZ1aX5aO0eZa/kLCXavzntVriL0rfKX3DarRj1BeFjkEwxGk5i2p6g7pVM4yfLvYO+WR5JNC6uIDneSSkf+9jH8LGPfSyxLSND6onUI1stJmWqU6abU16A1u5RtDNIzkptYhx1RWzdcrQmeiWzTzLFKtlyC0AdJDOTLEmSsi5h1BfEWDCEPKtF562iTJB5S/lJVyKTXFbsVGotM50YT93R54l7gQwZnwiS56ZxiIiaGE3tDbAmmWIz4pGzv4XO+EdSC+KqyeBIQKnJz1XD3rGocidm1ylWDJIpLj2RS1UVWVCPLMwpl4PksWAYPaqaa8p8Xn9QGX5TW65P5xJmkikeYUnCiFd+rSRXbmFTft7waG4HhSKLLDC7TrFikExxEZnkbKhHFmpUwRM7XGQXPRftCQySKR5ef1DJ/GpRbgEwc9rZF/2+Ppjjfw+KHYNkilkwFEbfkHwGXl6SPZnkKrcTFjM7XGSjtsiiPQCo1avcgkEyxUF0tgCSzCSr1oz053jmdGKQnOt/D4odg2SKWd+wX8lwVBRnTybZajGj0i3/Ply8l11EPXJxoS2pgCMZDjtrkil2orMFABQmlUker2fO9fKCLmaSKUEMkilmPVnW/k1N1CUzSM4uov3bXJ3qkQEgn5lkioM6k1yUxImdw2ZVenQP5Hiv5ImZ5Fw/aaDYMUimmEX3SM6ecgtA1eGi15PzK8GziahJ1qvUApCDFUBeGBoMcUQszWzYO57lTKa7BcA2cAAQCodxuj96QfZgDv89KD4MkilmorNFvt2KfIc+l65TRWSS/WMh9A2x2Xw2GBoNYDiSldOrswUwnkkGmE2m2YkeyRbz+LTGRHGgiNzbPxRp7VngkI/FXP57UHwYJFPMsrGzhTCHHS6yjnrR3txKHTPJqkCHQTLNRj2SOtnhN0qQnMPlFh2qUou6eSUAcjuzTvFhkEwx645M28umzhZCdakT4vOIdcnZobV7fD+Kcho9RGeSuXiPZqaMpE5i0Z5QUiSXa+RyUCgW7ZlM40GymLpHNBsGyRQzMWgjGzPJeVYLKiO/F9vAZYe2HjmTXFHigN2m3whaJ8stKA4ik5zMoj1BZJKHPGM5Ww8vFu2VFztQrurKxJILigWDZIqJLxBU6juzadqemii56GAmOSvoPY5acKiD5ACDZJqZqEnWomWhWzVQJFfbnolpe9WlBVFt8XL170HxYZBMMekZGF/Mlo2ZZEDVBq53FBI7XGS0sCShVelsoV+pBcCFexSf8XKL5DpbABOn7uVmyYXIJFeX5vPvQXFjkEwxUbd/K8/WIDlSt+r1h3gpLsP1DfrgjwzvqC3XOZNsUy/cYx0kzWzEI7/3aJFJFjXJQG4GhV5/EIOj8t+zutSJYg5YoTgxSKaYdEfav5kAlLmyu9wCGK9npczUqiqZ0TuTbLWYYbPKb7XMJNNMQuEwPD75NaJFTXJxgWo0dQ52uFAPEakuzYfVYlZOPpgIoVgwSKaYiExySZEdedbsfNlUl+VDNFxiG7jMJtq/WcwmVJfm67w143XJrEmmmYz6ghCFXlp0t8iz5nZQGBUkR64UipILDhShWGRntEOa68niHsmCPc+CssiiRLaBy2xi0V51mZw90ptTGU3NcguanlYjqdVyeeqeWLRnz7Moi/bEf3Px70Hx0//TgzKCKLfI1s4WgnrxHiWvZ9CrjIZOJ9EjWc9Je2rOSF2yj+UWNAPR2QLQJpMMAO6iHA6SI5nkqlKnMphFOWkYzb3MOsWPQTLNSpKknMgkA9Ft4NjhIjkeXxDbfv4abv75a2g5nb4a72AojM4+Y7R/E0Qm2cMgmWYwrMoka7FwD1BnTnMvKOxSdbYQxOK9XJ5CSLFjkEyzGhoNIBCUG9Fn47Q9NdHhYtQXxBAzDUlp7R6BLxBCWJLw6sHOtD3v6X4vgiH5BEfvRXvCeLkFg2Sa3oh3/D2nyJl8CzhgPHOaawv3wpKEzv7JQbL4e3DqHsWCQTLNSpRaALmTSQZYl5wsddvAN97tTltmvrV7PGtda5hMslxuwZpkmokot8izmmHL0+bjuSRSbuH1B5W2iLlgYNiPwJic3KkuUwfJHChCsWOQTLOK6pFcnN1Bco3qzbS9lx0ukqF+3XT1e9P29xSL9ux5FpQbpIaemWSKhSi3KHTmKTW0yVJP3RsYzZ1scoeqs0VN6XjyI3qgCINkmhmDZJqVqEe2WsxRzdizkdNuRZlLfhNlJjk53aopjYCcTU4HsVBwTnkBzBoFGsly2uQg2ccWcDQDkUnWqrMFMGGgSA6VXHSqTsor3ePJHU7do3gwSKZZiWCnosRhmKAjlWpEhwsGyUnpGfRGfZ2uIFmUWxilHhmIbgHHBaE0nRFlJLWGQbIqKOzPoaBQLNorKbQpxx8ATt2juDBIplmJYCfbSy0EsXiPbeCSI8otxFjmk53D6BvyzfQtSfOPhdDdLz/vXIO0fwPGa5LDkqTUSRJNpC630Ior36YkNwaGc6e8oHOKzhYAoqbuDXJxNs2CQTLNqltp/2aM+s5UE4v3hj1jGPLwTTQRgbGQUu93/llzlNtTnU3u6B1VJpYZZdEegKhMFtvA0XREdwutOlsAgNlsGm97lkOZUyVILpt8slzCNnAUIwbJNKNgKIy+yBtJzmSSVRnIDpZcJKRH1RFlxQI35lfKAWuqg2SxaA8A5hqw3AJgXTJNLxXlFkDuTZkLjIXQG3kPmmosPQeKUKwYJNOMeod8ECWU2d7+TZjDDhdJU9cjV5Q4sa6uAgDwbstg1FQxrYl65EJnHlwFxllkykwyzSYYCistArUstwBUQWGOZE5P93uVK0rVpZM/t3Ixs06JYZBMM+oZUPdIzo1yi3xHnpJ54eK9xKg7W5QXO1AfCZLDkoS3jvWk7HnbVOOotWqhpQUxlhoAfOyVTFNQnzwWaZ5JFqOpcyNz2qlq/zZjJjlHThoocQySaUbdg7nTI1ltDjtcJEXUsRcX2GDPs2BuRYFykpXKkgvR/s0o46gFdSaZvZJpKiMpGEktqMstcqG7igiSrRbTlJ9b0VP3uJCWpscgmWYkgp0ChxX5Dussj84e7HCRnPHFnvIHlMlkUkouDh7vg39M+2zqqG9MGb1rpPZvAINkmt2wN4VBcmTqXiAYzonXnwiSK935MJsnX1GKnrrHbDJNj0EyzahH6ZGcO1lkYDyTPDgSwKgvdTW02UrdW1uoXyYHyYFgGAea+zR/TvWiPaMFyXabBeKjOheCFIpfdLmFtvX07qheydlfcjFd+zchaqAIF+/RDBgk04xERrA8R4NkAOjo4eK9eEiSpJTpqE+ultYWwxWptUxFyUVbZNEeINckG4nZZIIjkk3mwj2ayoiq3WShU9urdlFBYZbX4UqSpEzbq5pi0R4wYaBIlv89KDkMkmlGopVXrizaE9RBMksu4jPsHYM/IJdTqOsBzWYT1kayyfubehAMaVsL2BqpRy512ZHv0PZytRbEQBFfgAv3aDJRbmG3WZBntczy6PiIcgsg+zs6DHvHlBPR6TLJxQXjfw8OFKGZMEimaXn9QeUSYEUOLdoDIi3EIllPLt6Lz0wdUdbVlQOQF8y82zKg6fO2nY6Moy431qI9wclMMs1ALNwr0rgeGZDXlFgt8sd9tgfJnaq2nTWlU19RyrOOT93L9r8HJYdBMk1LlFoAuVeTDLDDRaJmet2sXFCqjKnWsuRCkiSls4XR6pEFp00Okn0MkmkKyiCRFATJJpNJNWUuuzOnUe3fyqbOJAO5N2CFEsMgmaYV1es2x8otAKAmEiS3GThIDksSdr7wLr59z18wZJDLhiJItlpMUZd5ATmDs2ZJGQDgzaM9mrWjGhgJYNQnB59GmrSnJjLJXLhHUxlO0bQ9QRyL/VkeFIogudCZN+MJR3GO9Y6mxDBIpmmJqWkmE1Dmyr0gWbSB6x/2Gzawef61FvxhzykcOt6Hv+7v0HtzAIwHyWXFTpinGOghulz0D/txonNYk+eMXrRn1HILOYPuZU0yTSGV5RaAeqBIlgfJsyzaE5hJplgwSKZpiWCntMiu1LPlEqMv3jvWNogn/9ikfN1hkG2cbbHnmiVlsFrk4FmrkovWSPs3kwmomeESq56YSaaZjHjljGahMzXj1N05EiR39c/c/k0QJw2DzCTTDHIv8qGYjQc7uVePDEwIkg1WcjHiHcN//e8BhFXlCuoFK3qaOEhkIqfdipULSgFoFyS39ciZ5Ep3Pmx52nYG0IqoSWaQTFNJfbmFHHwPjgSi3jeySSgcxul++f0n1iB5xDvGqXs0LQbJNC2lR3KOdbYQXPl5KIhMGTRSr2RJkvDz5w6hbyh6upx6wYpegqEweociJ1czvG7qI10uOno9mmTAxSARo9YjA6pyCz/LLSiafyyEwJgcqKW63CIUljDsyc4BST0DPoTC8glA9TSdLYSoqXuj2Z1dp8QxSKYphSUpZ3skCyaTabzDhUFKGQDgD6+14K2mXgDAe+trcenZ8wDI/T49Pn2zlH1DPogk1Uyvm/plFcoEumSzyeGwpGT6jTZERE0ME/GPhRAOZ2cmjxIzmsKR1EIuDBTpiLGzBTC+cA/g4j2aHoNkmtLgSEC5BJVr0/bUjNYGrqltEL/+k1yHPK+yEJ9639Koy4qiHk8v3YPqHsnTv26KC2xYMrcYgNzlIrnn9CIQea3OrTDmoj0AyLePT1HzBlhyQePUmd2iFJVbuHNgoIgoOTOZgMpZPreiMslZ+veg5DFIpimJzhZA7tYkA+MdLnoGfcoUOb2IOuRQWILdZsGXrzwTeVYLasrGs6d61yWreyTPVqazLtLlorl9CP1JZLZEqQVg3B7JwPjCPQDw6pzxJ2MZSUMmubhANYo5S4NCkSQoL3YgzzpzeKOeusdMMk2HQTJNKWogRHFullsA0Yv3Ovr0yyZLkoQH/+8d9EbqkK+9fLmSQS4utCn1rnrXJYvXTaEzD/kO64yPFdP3AODNo4mXXLRG2r9ZLWZUuo17Que0jS8oZBs4Uhv2jgdphfmp6W7htFuVQT7ZGhSKJMFs9cgAp+5RbBgk05TEaGGb1QxXQWretDOBUTpcvPD3Fuw7JpclXLB2DjadUa3cZzKZUBspM9A/SJZfN+UxnFhVuvOVhXZvJlGXLDLJc8ryYTEb9y3NqTppYIcLUhtRlVsUzHJymQxRl5zMlRsjE+9/s3W2EIrZK5lmYdxPFNKV0tmixAnTFAMhckVJoU25TN6uU4eL5vYhPBHphzy3ohCfet+ySY+ZY5AguWeW9m8TicEih08NYNSX2Ip7o4+jFkQLOIBBMkUT5Rb5dmtKe9KLuuRsDAq9/iAGI1NHZ1u0J5Rw6h7NgkEyTUkswIolI5jN5A4X8huuHpnkUd8Y7nsmUoecZ8GXr1w1ZR9gsWCtq9+jaw/U2XokT7SuTg6SQ2EJ+4/1xv18Y8EwuiInBkZetAdMqElmkEwqqe6RLKR6ytxYMIz/e/kE3jnel5KfPxN1gqA6xrIr8ffgwj2aDoNkmlK8wU42E4v30t0GbrwOWT5h+dzly6MW6amJTHJgLKxbeyePbwyjkQVp5TG2DZxfVaiMPH8jgbrkzj6P0hfV8JlkO2uSaWqpHkktpDpz+sxfm/E/Lx3DbQ/tQSic3gEdUUHyNO+TEzGTTLNhkEyTjAXHA61cXrQniLrk7gEvAmPpC252vd6qtEc7/6w5OHdV9bSPra0cz6LqVXIh6pGB2E+uTCaTMljk7ebeuP++bZFFewBQW27sTHKe1aKM42YmmdREuUWqOlsIIigcHg0gGNI2iPUHQvjTm+0AgKHRAE50DGv682cjFu3ZbZao9m4z4dQ9mg2DZJqkb8gHccGemeTxIFmS0heAHu8Ywq92HwMgT5G75uLJdchqtRX6B8mJtg1cHym5CIyFcfBEfJdpRT2y025Bqcs+y6P15+BoapqC6JOc8nKLSE2yBDmQ1dLLBzvhUb2uD5/s1/Tnz0ZZtOfOj3kdjbotHqfu0VQYJNMkUb1uGSQr5RZAekouPJPqkM+csg5ZzWm3Koty9M4km00mlBbFHrAunVusZNDefDe+wSKis0VteWFGLDAVA0UYJJPaSKQFXJEztZ2E3Kopc/0a1uFKkoRdr7dE3faOTkFyVWnsn1klqvepQZZc0BQYJNMk6qlpub5wDwBKXXbYI/1FH991FC/ubdX8UqUgSRIe/O1hZST45y6bvg55ItH2SL8gWT65KnXZ41qhbzGbsXapXHKx71hPXLWMokey0euRBUekLtnrZ00yySRJGi+3SNPCPQAYGNYuKDx0oh8dveODPADg3ZaBlL1PThSWJGUBb6zt3wCgJAcGrFByGCTTJOqBEOoV+bnKZDJhy+oaAMCQZww7X3gX3/3ZHrz2TpfmnSRe3NuKNyI9g7esqcG5Z05fhzyRaHvUpXOQnEiJjuhyMeIdw9GWwZi+x+sPKicTteWZESQzk0wT+QIhBEPy+0iqa5KLC1MzmvqFSBa50JmHT11SB0D+vU52pqcueWDYr4ymj7X9GzDx78FMMk3GIJkmibfXbS645pI6/Msn1mJ+lVz7e3rAi//634O47eHX8U6cdbTTOd4xhP95Sa5Drq0owDWRD5tY1UQ+HHoGfbosQhFXICpi7GyhdsZCN+yRkpJYu1yoS1+M3v5NcDJIzhivHz6NvUcSH3ITK/VI6lR3t0jFlLmuPg/2N8ntG99bPwdnLiqF2SyXPh0+lZ6Siw5VYqAmhml7Qp7VrAxvYSaZpsIgmSYRtaWJBDvZbNWiUmz7/87G1g+foVxSPNE5jMbH9+HO/9mHU12JZ03Udci2PDO+/JEzlaAxVmIUqyQBp/vTm00OhyX0DiZ+cmXLs+DMxaUA5Ol7UgwZelGPDABzMqXcQizcCzBINrK9R07j3mcO4N6n31YWh6aKOkhOdbkFoGp7plGryF17WwEAFrMJF9bPhdNuxdK5xQDkIUHpIDpbAPHVJAPjdckMkmkqDJJpkp4kgp1sZzaZsOmMavxw6yZcc/EyJStz4Hgfvv/Q3/GzZw8qmfhYSZKEh343Xof82UuXR43DjlWN6jJjZ19825CsgRG/csm4vDix140ouegd8uNU18gsjx6vR3YV2ODKz4zR6Sy3ML7AWEi5oiMBONo6kNLnG1aNpE51uQUAlBRpN1DE4wvir293AADWL69QFg+vXiKvMTjamp66ZLEOo6TQppyIxkqcNHDhHk2FQTJFiRoIwUV707JazLh4wzzc8aVz8aH3LIQtzwwJwCsHu3DTz17FY7uOYtgT25vuS2+0KZd1N6+uwXmR+ud4lZc4YIlc5uzsS+/gE3VHlERPrtYsKVO2X9Rlz2S8s0VmZJEBLtzLBH/4e4tywgog5f1+RWcLAChKw8melgM0/vZ2B/yRwTiXbJin3L46shA3MBZOS7/kzgQW7Qli8R4zyTQVBskUJZGBELnMabfiqvMX444vnosL62thNpkQDEl44fUW/Ov9r+C5l08oHyJTOdE5hP956SgAOdj79KXx1SGrWcxmVEbGsXalOZMc/bpJ7OSqwJGHFfNLAMRWlywug2dKPTIQnUmOpaSE0qtvyIf/e+VE1G2pXnwmpu2ZTOOvj1QSQXJ/kuUW4bCEFyOlFotqXFhSW6zcd8aiMphN6atLVjpbxNgJSG283IKZZJqMQTJFYY/kxBQX2vHZy5bjtuvOwYblctmA1x/CU39uxr/+9BX8cV/bpNZmHl8Q9z1zAMGQXIf8pSvjr0OeqMqtTxs48bpx2CxJXTIWJRdt3aPomqGuesgTUIYhZEr7NwBwRIKgUFhKW3ssit2Tf2pCYCwMkwnYsKISgFzWk8qFsMORmuQCR56y4C2VREmExx+EP4kJovubenE6ctxfsmFu1H1OuxWL5rgApD5IDoyF0BvJ/CeSSRYDRUa8YzwmaRIGyRRFXGaMdyAEyapL8/H/rlqN73xuPZbPKwEg17o98vsj+N4Dr2HvEXlRmiRJ+MXvDysZ2M9eulyTsgHR/ijtQXKkjr282JnUUI+1yyqU/59psIh60V4mBclO+/hJkIclF4ZyrHUQrx7sAgBccNYcnL9GLnsKhSWl/j0VxMK9ojQs2gOieyUPJlFiINq+FRfalBMKtTMWuAHIf9dUBp+n+73KhNiEyi0KOVAk3bq6utDQ0ICNGzdiy5Yt2LFjB/z+mV+Lra2tqK+vx549e6a8/7vf/S5+8pOfRN3m9/tx0003YcOGDdi8eTMefPDBuLeVQTJFSXQgBEVbMqcYN1xTj69/bA3mRoK4zj4P/vPpt/HDX+7F/7x0DK8fPg0AOG91dcJ1yBOJD4kR71jUqvlU69GoI4q7yI7FkQzUTHXJbaqgZU4Cl1j14lQtKvJx8Z5hhCUJO3e9C0Auebjy/MVYUF2k3J/KkgtRbpGORXtAdFCYaIlBa/eIMlHvwvraKT8rVkSC5EAwjOb2oYSeJxbqhEB1nJ0tgIl/D9Ylp5okSWhoaIDX68XOnTtx1113Yffu3bj77rtn/L7t27fD45k6+fOzn/0MTzzxxKTbf/SjH+HAgQN4+OGHcfPNN+Oee+7B73//+7i2l5MiKEo3O1toxmQyYc2Scpy5qAyvHOzEM39pRu+QH03tQ2iKfGjMKS/AZy5ZrtlzqjMpXX0eFKrqBFMpmUEiE62rq0Bz+xCa2gYxOOKPavgvtEYyyeXFjowaeKPeVg+DZMP429sdSiD8kc2LlG4pZS47eof8OJHCIFmUW+gRJCdalyxqka0WE967tnbKx9TNK4HFbEIoLOHwqX7URa6saU30SLZaTAl11omaQsggOeWam5uxb98+/O1vf0N5ubzAs6GhAXfccQe+/e1vT/k9v/nNbzA6Onkx+sjICG666Sa8+uqrqKmJTjR5PB488cQT+NnPfoZVq1Zh1apVOHr0KHbu3InLL7885u1lqpCiiMv/7GyhHbPZhPNW1+CHWzfh4xcuVZrX26xmfPkjq5SR11pQB8npKrnwj4UwGKkP1iJIrl8mv3FKAN48OnXJRVuPnEnOpEV7QHSQzEyyMXj9Qfz6T80A5DaKF64bD/oWVstXNU50pi4Tmu5yi+ICG0RFVCJB4Yh3DK8c6AQAnHNGFVwFU3fksNssWFQj//2OpLBfsli0V+nOT6imuzgqSGa5RapVVFTggQceUAJkYWRk6pKm/v5+NDY24pZbbpl0X2trK/x+P5566inMmzcv6r7Dhw8jGAyivr5euW39+vV46623EA7HXv5j6BSM2Wya8UVviVzisbAsQBNhSUJxoQ2FzjysWOCG1Tr735X7IHZWqxkfPG8hLt4wFwdO9KG2rBA15fHX0E2k3gdulx1nLCqFPxBCMCzFtA+T1TfsU1a2L57jSvo551UV4ZwzqtAz6ENnv2fSz5MkCbY8C5bUFuOsZeVp+R1nE+tx4HbZlb+V1WoxxLZnk0Tej155swsVJU5UlDjx6UvrlMWVALBueQUGRgPK51Aq9pe4GrKwpjhtr4c1S8sx4hlDXp457uc8cKIP86rkUpQPb1486fvV+2DzWTWQAIQjRcOp+P2sVjOW1BbH/Jk11fevWlwKnz8EqzX+v4cRGflz2eVyYcuWLcrX4XAYjz76KDZt2jTl42+//XZcddVVWLZs2aT7VqxYgfvvv3/K7+vu7obb7YbNNn4SVF5eDr/fj4GBAZSWlsa0vSbJwH2IJElKahEQERERERnTHXfcgZ07d+LJJ59EXV10C9SXX34Z27Ztw3PPPQeHw4Hly5fjkUcewTnnnDPp53z2s5/Fxo0b8dWvfhUA8Mwzz+A//uM/sHv3buUxLS0tuPjii/GnP/0J1dXVMW2foTPJfX2js2aSXS4nhoa8CLF1S9JOdg7jF787DAD4pw+uxLwYLmVzH+hv4j546s9NeLupDxVuJ/7flWem/PlfPdSFP+w5BQD4zmfXa5KJaesZxQPPHgIAXH3BYqxeXKbc927LAB7bJfeW/tJHVqEqgRXtWov1OAiHJdz68OsAgMvOmY9NZ1SlaxNzQrzvR//9wrs42joIi8WEL195Jspc0WVmo74x/Ntj+wAAHzh3Ac6eootDMrz+IH70328CkGuh1y4rn+U7tPHYrqN4t2UA86uK8I8fWBHz9x083ocn/9gEAPjYe5fgjEWTs3HqfeD1B3HHzjcQDku4YG0t3ls/R7PfAZBLP/798X0AgA9vXoh6VXecePzyD0fQ3D6EJbUufOZS7daI6EWPz2W3O/4F1I2NjXj44Ydx1113TQqQfT4ftm3bhptvvhkOR/zln3a7HYFAdPmM+Dqen2foIDkclhAOz57oDoXCCKawj2WuONEhL5YCgAKbNa6/KfeB/sQ+MElAU9sgTnYOI3BFSGnqnyrvnupHU9ug0n9Vi9dBZbEDfUM+9A/78cc32rByvlu57/BJ+fksZhOKC2yGet3Fchy0do/AHwihvXsEwWBiH+p6CYbCeP3IaSyZU2zoxb2x7Ie3m3vx+8jJ3fs3zUdx/uTXkt1qwcCwH71DPrx1tBv1S7UNYvsGfcp7bjrfQ73+IJraBjHsCcT1nL/+UxOa2gZR5rJjSa1rxu8NhcLyoidJQlPbIPIsJmxeHVv2LlanOoeVv58zL77PLDWPdwxNbYNymZqB3k+SZeTP5VtvvRWPPfYYGhsbcdlll026f//+/WhpaUFDQ0PU7ddddx2uvPLKKWuU1aqqqtDf349gMAirVQ51u7u74XA44HK5Yt5OQwfJlF6iQ4Etz5y2RSSkPZFZDYbC6Bv0pXwoTE8KFnuaTCasW1aBF99oxf7mXowFQ8izygscRfu3qtL8jGxT6LRZ4A+E4MvAPskvvN6CJ3Y3YW5FAW75/ORLnpkiGAorVyOKC2z44LkLp33swuoi9A75UtLhYljVprEwje+57sLxUcyxljWe6BzCsVY5IL1o/VxYzLEdeyvmu3G0dRDH2oaijmMtRLV/K0v8ilKxMqqb3S3S4Z577sHjjz+OO++8c9pOE2vWrMHzzz8fddull16K2267Deedd96sz7Fy5UpYrVbs27cPGzZsAADs3bsXq1evhjnG1y7A7hak0q30uk1uIATpK6rDxQxT67SSqraB9XVy1s4fCOHQifGpXaL929wMGiKiJjpcZGILuLci3UZau0cx5MncTgAv7W1VAqyPXrBkxjaCC2vkRWpt3aMYC2p7YiN6JANAUZpawAHjbeACY2F4YzxZ2/W63PbNlmfG+WfFXjYh+iUHQ2E0tWnbJUTsw0JnXlIt9EQbOE7dS72mpibce++9uO6667B+/Xp0d3cr/wA52+vz+eBwOLBgwYKof4CcIS4rK5vpKQAATqcTV155JbZv3479+/dj165dePDBB/G5z30uru1lkEyKHhHsJNBrkoxDXaPb2ZvaIFmSJE17JKvVzStR2uW9eVR+Aw2Fw+jolYNkLSYU6kEEZL5AZgXJ/rGQ0t8bAE50pK53cCoNjQbwv387AQBYVFOE98xSAiCGisiT9yb3ak3GsHf8RKPQOXUrtVQoKYpvgMbgaACvvSNPI3zPmTUocMQekC6Z44LVIiddtB5RLd7fEpm0p8ape+nz4osvIhQK4b777sPmzZuj/gHA5s2b8dvf/laT57rxxhuxatUqXHvttfj+97+Pr371q7j00kvj+hkstyCFGEldnuTUNNKXPc+CUpcdfUN+dPV5U/pcQ54xBMbkzEuy0/YmslrMOGtpOV4+0Ik3j/bgc5dJON3vRTAkr1OozbAeyUKmZpKPtQ0ipFojcqJjCGuWzJ7RMZqn/9IMb+Rvf83FdbPW7IteyQBwonNY6f2rBdEj2WI2RY0sT7WogSIjfsyZ5YTzj2+2Kcfd+9bPjeu5bHkWLJlTjCMtAziscb9kkUmuSmDSntrEqXtlnBOQMlu3bsXWrVunvf/IkSMJ3ffLX/5y0m1OpxN33HEH7rjjjvg2UiXuTHI8M7ePHDmCT33qU1izZg0+9KEP4dVXX014Qym1xoIhZfoSM8mZT2RWOvu0zXxNJLLIABKadjUbsVp92DOGY22DaFNl8jK23CIyPCbTapIPn4zOAjZ3pG7ARqqc7BzGn/e1AwDOXVWt9KyeSaEzT6m3P6Hx76weSZ3OEje3OpM8y9S9sWAYu99sAwCsWuhO6ArO8vklAIDm9kEExrR53YfCYeX9J9lMMgeK0HTiCpLjmbk9PDyMf/qnf8LSpUvx7LPP4pJLLsH111+P3t5erbadNCSyyABHUmeDKiVITm0muUcVJKfidXPm4lLYIi3l3ni3G62RRXs2qznlCxJTRWSSvRmWSZ54qfxExxAM3GZ/EkmS8NiudyFBvtryD+9dEvP3ipKLkxov3lNGUqd5oXSBw6qUQMxWbvH64dMYikzUvHjDvBkfO52VSl2ypHSjSFbPgE+5slFdmtwJM0dT03TiCpLFzO0dO3Zg2bJl2LBhAxoaGvDcc89NeuzTTz+N/Px8bN++HQsWLEBDQwMWLFiAAwcOaLbxpB2xaA9guUU2EJmVviGfZpmbqYhMTp7VHJWN0Yo9z4JVkV6scpAsZ5LnlBekvLVdqihBcgbVJPsCQaUGWby2hjxj6BvKnIDi74dP491Id4YPvmdBVDZ1NgsjQXJbj7aL90QmOZ2L9gC5e0yJ0tFh+sypJEl44fUWAECV24nVCZbXLJ7jUjrRvKNRyUWHRp0tACDPalHWPwyOZs5rmlIvriA5npnbr732Gt73vvfBYhmvs/r1r3+NCy64IMFNpVQSi/YAlltkAxHISAC6+lOXTe5WtX9LVdC6rk4uuegZ9OHg8T4AQG2GlloAmZlJPto6Xo98+TnzlduPZ0jJhX8shF/tPgZAfq1eenZ8GVFRlxwKS2g5rV0Jk6hJTqYzQ6KUIHmGcoumtiGl9d371s9N+BjPs1qwtFb+Gx7RaPGeWLRnMgGVGlxVGv97sNyCxsW1cC+emdstLS1Ys2YNvve97+Gll15CbW0tvv3tb2P9+vUxP5/ZbJp14p76v5S43iE52HEV2FAQx6U/7gP9TbUP5qoWtXUPerFojnaLjdTEyVWl26nJpL2prF9RiYd+exhhSYI/khWfX1WUsudLRDzHQUEkIPL5QzBbTBmREX+3ZQCA3P5r81k1+J+XjsLrD+Fk1zA2nantgIhkTLcffvO340rW+5pL6uCMozsDACyuHT9+WrpHUBepsU2WCJJdBba0v55LXXagDRgYDUz73LvekNu+OWwWXFBfG9M2TrcPzlhYisOnBtDcPoSQJMGel9xCxdMD492YnI7kexC4i+xo65FbGxrpvSUR/FzWTlKvrMbGRhw6dAhPPvnkpPs8Hg9++tOf4nOf+xx+9rOf4f/+7//w+c9/Hr/73e9QU1MT088vLS2IaTGDy8XMZ7IGRuU365rygoTGS3If6E+9D1zF+cizmjEWDGPAM5bQPo1FbyQLNa/KlbLncLuBM5eUYf+xHuW2FYvLU/Z8yYjlOCh3j2f5nfl25McZsOlBlCmcsagMVRUuLJvnxv5jPWjpHjX8fjjd78FvXzkJADhrWTku3rQw7kVybncBKkvzcbrPg/Zej2a/86hPft+tKE3sfTcZ1eWFAE5jyBOY8rm7+714/fBpAMCl5yzAnOrZFzmqTTwWNq6eg6f+3IxQWELngA9r65Ib8d0TSezMqy7S5G9XWVYAHO/DUArfL9ONn8vJSzhInmnmNgBYLBasXLlSGSl4xhln4G9/+xv+93//F1/60pdieo6+vtFZM8npnk+erdpOy5fU3IU29PfHfjmR+0B/0+2DSrcTbd2jON46GNc+jVUwFEZPpJTDlW9NyXMIa5aURgXJJc7UPl+84jkOpNB4TWt755Dh2015fEEcax0AACyrdaG/fxTzKguw/1gP3j3Vj96+EcNkw6faDz996m0EgmGYTSZ84sKlGBhIrHf4gqpCnO7z4MjJPk1ee6FwWKlJtpqR9tezM0/OMvYN+qbch0+/dAzhsAQTgC2rq2PevumOhYoim3Li/tqBDixIsmSqpUv+zCp32TX52+VHus70DnoN9d6SCD0+l7PlxGKihILk2WZuA3L98uLFi6NuW7hwITo6OmJ+nnBYQjg8++ppI88nzwTqgRDlxY6E/pbcB/qbuA+q3Plo6x5FR+9oSvZNV58H4ugsK0rsdROrtUvK8SjeBSCvzC90WA35eovlOLCpxvKOeAIoLkjfEIlEHDrRB9HEom5eCYLBMBZUygvZfIEQWrtGZu2zm25iPxw51Y89h+QhGBfW16K6ND/h1838ykL8/Z3TaOsehdc3lvR45SFPQDl+8u3pfz278uXXXSgsYWDID5fqdRgYCylt385aWo4yV/zH98RjwQRgaW0x3jnZj0Mn+pL6fb3+oDL0o9Kd+D5VK4qUGQ57xuDzBzNy5P1E/FxOXtyvAvXM7SuuuGLax61du3ZS4+fm5mbU1tbGv5WUUqO+oDKaNBW9bkkfSq/kXk9KWnVF9UhOcTu2UpdD6TAwt6Iwo8emq0cgxzoSWE+iP7LDZlH2gXqghlEX74XDEh7bdRSAfGL1kS2Lkvp5Wi/e02sktTDT1L1XD3Up9dIXb4hveMhMVkRquU90DCc1cbJT3dnCrc17j5tT92gKcQXJsc7cBoBPfvKTOHLkCH7yk5/g5MmT+I//+A+0tLTgIx/5iPa/BSUlqrNFhvaepclEkOzxB5V+rFrqVvXWLk9DycAn37cMa5aU4cokgx29qSerZUIbONEfuW5eCSxm+SOj1GVXMo9GHU/95/3tOHVa7rx01fmLk+4gIXolA8CJzuRPDEZUx2S6+yQD0/cGVrd9qy0vUHoca2FF5GeFwhKOtSbeL1l0tgCA6jJtrmJETd1jGziKiCtIjmfmdm1tLR544AHs3r0bH/zgB7F792789Kc/RVVVlfa/BSVF3SO5wuD1kRQ79RSqrr7E6jBnIjLJRfl5UdnRVKmbV4Kvf+wsLJ+v3Ye2HqIzycYOkke8Y2jpkgPNFaq/u8lkwqJI0GjEyXujvjE89admAPJkxgvWzkn6Z0ZN3tNgqMiwKpOsZws4ILpX8uGT/cpky/dtmKvpVZtFNS7YIrXQyYyoFplku80SFewnI2rqHtvAUURcn2zxztxev349nnrqqcS2jNJGTE2zmE1wu2JvsE/Gpm6w39nrwbK5JZr+fBEk8+pDfJy2zAmS320ZUOpmVywoibpvUY0LbzX1ouX0MIKhsKFqOJ/5c7OSqf3U+5YpGfBkLawuQs+gT5PJeyPe8UCsyJn+unSn3Qq7zQJ/IBTVK3nXXrntW4HDinNXadvez2oxY1ltMQ6e6J80wTEeIkiududrFsSrg20OFCHBOO9qpBsR7JS67Jp9mJD+Cp15yhSpzhRkkntUg0QodrY8s9JJwOg1yaIeOd9uxfzKoqj7FkbqkoMhSRkXbgQtXcPY9boc6K2vq8DKhaWa/WzxO7d1jyY9yVIE8XlWs5JdTTeRTe6PlFucHvBi31G5i8z5a+ck3ct4KuJK0ImO4YRPEpUgOclJe2rqqXscTU0CIyJSakuZEcw+4kMkFUEyM8mJMZlMSl2y0TPJ6nrkie04F9WMB83HDVKXLEkSHvjfAwiFJVgtZnz8oqWa/nxRlxyWJLQkeWIgyi0KnXm6LUR1R7KnIpP80t5WSADMJhPet067BXtqoi45LEk4mkBdcliSlPKxKo0W7QmxjOqm3MIgmVTt3xjsZJtqd2qC5FHfGDyRAI9BcvwyYTT1kCeA1kht6oopFm8V5duUqwjH241Rl7zvWA/eOCIPwLj8nHmavzYXVI2fGCRbciEyyXp0thBEh4uBkQC8/iD+sr8dALBueQVKXam5QrSwukjJUCcyorp/yI9ApK2ZlplkYLwumZlkEhgk57hwWEKvkknmZfNsIz5ETvd7Y+o5Hit1+zcGyfFTgmQDd7d4V7WwasU0Y5hFK7jjGnR7SFYwFMZ/Py/30nYX2fGBTQs0f45CZ57yPplsVw8RJOvR2UIYz5z68fKBTqX85+L1qckiA5G65Lny9L5E6pI7+8dP+GtKte3PLf4ebAFHAoPkHNc/7EcoEjwx2Mk+VZFMcigsRbX6S1YPO6IkxWkT5RbGrUl+JxLAFDrzMLeycMrHiCC5vWcU/oC+v8tLe1vRFZkA+YmLlsJhS03HlQWRfsnJdrhQl1voRQSFQ6MBpe3bgqoiJYhNleWiX3Jn/HXJ6vZvVaXafmYxk0wTMUjOceyRnN2iOlz0aRckd7MjSlIyodxCLNpbPq9k2rHToi5ZkoCTXfrWJe95R56st3ReCc49U9uuDGpioEp7T3KL90R3Cz06WwjuSLmFBPlqEyAPD0l1jbQo35EkuYNKPETpmLvIrvmJkDhpGPaMIZimcc5kbAySc9xp9dQ0ZgSzTpXbCfFxp2VdsgiSy1wOdkRJgNGD5MERPzoiGbup6pGFBdVFEPFUs451yaO+MaX84bw1c1Ia5C1UL947nfjiPWOUW0QH6K4CGzauTP0sgwVVRbDbRF3yQFzf25miRXtA9NS9oVGWXBCD5JToG/Lhd3tOajKVKdXEZXOHzaLrZT9KjTyrBWWRkx8tB4ooiz1Zx54QowfJh2OoRwYAh82KOZGJZ3q+3x0+2a/0c15bV5HS54qevJdY9jwYCiulNkYotxDeu3YO8qypDwusFjPqIn3b34mzLlmUW2g1aU9NPVCknyUXBAbJmuof9mPn8+/iX+9/BU/sbsJPfv02JEm7xVKpIMotyoudurUhotQSk/c0zSSzbWBSHKIFnM51vNMRC6pc+XmYUz5zMKIs3tNx8t7BE/L2FjjzsHhOautpCxzji/cS7XChHkldZJBMssVswoX1tWl7bnHydaprGB7f2MwPjgiMhdA3JL/3qCeKakV90sDFewQwSNZE/7AfO194F9/+r1fw4hutCIYk5XaxkMSoxEhqdrbIXlUaB8nRHVEYJCci3+iZZFGPPN8968mzqEvuHvBh2KNPYHHoRB8AYNVC96R+zqmwUFm8l9iJwYjOI6mFPKsFxQVyoLxxZSWKC9O3viC6Ljm2fsmn+73KFYPUBMmq0dTMJBMYJCdlYMSP/94lZ45f3NuKYCgME4D6ZeXKY462Dui2fbHoHuRAiGwnPkz6h/3wadByrG/Yx44oSRILjsaCYcMtEFKf3M9UjyyIKXRA8h0fEtEz4FUWna1apN10vZmML97zJLR4b9hrjCAZAD59SR02nVGFj12o7eCV2cyvKoQjUpccays49Ym+1j2SgYlT95hJJgbJCRkc8ePxF4/i2//1Cna93oqxoBwcb1xZiVu/cA6uv3q1smr4WAIThdIlMBZSLilx0V72UmdcujTocNGtav/G101iRCYZMF42WWSRgZnrkYV5lYWwWuTsrR4lF4dU25uuIHlBkov3osst9OtuAQAbVlRi64dXTapPTjWL2Yy6eSUAYg+SOyJBstViQnmKhp0Uq3pHE6WmkWSWGhwN4HevnsQf32xTJv6YAJy9shIfes9C1FaM9xJdWluMvx8+jWNtxg2SewZVvW6ZEcxaUUFyvydq4VEiejhIJGmiJhmQ65KLtE+KJUwspCoutMV0SdtqMWNeZRGOdwwlPWAjEaLUoqLEgUp3ev6QExfvLamNrw56RFWWUujM3Y/hFfPd2N/Ui5auEYx4x2bNqotFe5Xu/JSV1ZQU2tDeM8qaZALAIDkmQ6MB/H7PKbz0ZisCY+OXRjesqMSHz1uIuRWTG+0vnSsHyR29npgOfj2op6aVM9jJWm6XHTarGYFgOKoRf6JEiY7TblUuTVJ8ojLJPmNmklfGUI8sLKqRg+TmjiFIkpS2RcBhScI7ke09Y2F6ssiAvHivssSJ0wPehOqSRbmF3WZBntUyy6Oz14oFJQDkPs1HWwZQP0tnkq7ItL1U1CMLxQXMJNM4fsLNYNgjB8cvvhEdHK9fXoGPnLdo2ilUAKImFh1rG8TapeXTPlYv6kwyL5tnL7PJhEp3Plq7R6JGuiZKvdiTHVES41QFyVrUiWulZ9CrvC/EUo8syB0u2jA0GkD/sB+lKboUPlHr6RFlcl06g2RAziafHvAm1OFCLNwrMmDyJJ3mVxbBabfC6w/inVP9MwbJkiSNt39LYZBcUsSpezSOQfIUhj0B/OG1Fry4txV+1aKM9XUV+NB5CzG/avbL1XMrCmHLMyMwFkaTQYNkkUkuLrDBnpe72YxcUF3qlINkLTLJkddNRTGvPiTKoQqSPQaqST58ckD5/1jqkYVFqsV7xzuG0hYkH4q0fjMhvu3VwsKaIvz98Gm09YzCPxaK6z1UGSSS40Gy2WzC8nkl2HesZ9ahIsOeMeVYSWmQPGHqntXCpVu5jEGyyoh3DH947RR27W2FX9W/tH5ZOT6yeVFMwbFgtZixuMaFw6cGcNSgi/eUYIelFllPrATv6vckfTm8h6+bpKnLLXx+4/RKFguoSl32uPZvdVk+HDYLfIEQmjuGsH55Zao2MYqoR55fVZT2BXALq8ZHcrecHsHSOOqShw0wbc8ols+Xg+SW0zPXJUd1tkhDkAzIpZbpOuEjY2KQHPHKgU788vkj8KmC47VL5eA40YVOS+eW4PCpARzvGDLkGam4bM6padlPfKh4/SEMjQYS7ofqCwQxFLlUzN7aiROtrwDjZJIlVX3vijjqkQG5pGdhdREOnxpI2+K9sWAY77YMAADOWBh7aYhW1J8LJzuH4wqSWW4xbsX88X135NQA1i+fuuQi1e3fhOheyQySc52xojYdPfvyCSVAPmtJGbb9fxvQ8A9rkuoEIN40x4JhnOxK/6rvmUiSpEzb42Xz7FelyrwkM1SkZ4AdUbRgtZhhi4z/NUoLuNMDXvQPy3WY6sAlVqJf8onOIYTTMGm0qW1Q6TJ0Rppav6nlO/JQ6ZaPgRNxtr4b8cqdEwqd+rZ/M4J5lYXKlZWZWsGJ961CZ15Ky1TUCQTWJRMzyRGfungZ3m7uxbmrqqPq65KxtNYFE+SVu02tg1iS4nGp8RjxjiknBcwkZ7/qCUHy8gSCIIAdUbTksFsRCAbgNcjCvaj+yJGuA/FYHHnf9PpD6OrzoKZs5nHWyToYKbWwWsxYFmcLNq0srC7C6X4vTsSZBGG5xTiz2YTl80vw5tEeHJkpSE7Doj0AKCkYP3EZZJCc85hJjli9uAzXXFynWYAMyJmGORXyB8VRg/VLVne2qGSwk/UKHHkoinwgJ5NJ7o68bkwAyngZMilOZTS1MWqSD0cWTpUXO1CewNWlhTXjV93SMVRELNpbNrcYNp0WHi9QJu+NRi3ynol/LKR0S2K5hUyctLd2j2JomtHm4n0r1UGyLc+iZLb72Ss55zFITjGR4TjWOggpDZcgYxWVEWS5RU4QHy7JTN0Trxu3y448K98+kuGM1CX7DFBuIUmSkkmOp/WbWpnLoZyIHU9xXfKob0zpT6xHPbKwsFpOqkgS0NIV2+S9UQONpDYKdWeSd6fochEMhZX3nqrS1H9elUQm5jKTTPyUSzExiWlwNBCVvdWbeMOxmE3KCG3KbqIuuSOZTDLbv2lGZJKNsHCvs8+DwVE5a7YywVIck8mkXImLt0Y3XodPDkDkHNLdH1ltQZV68l5sv7Po6wxAOanIdXMrC5XBRFPVJfcM+hAKyzu8ujS1ZTzA+OK9AWaScx6D5BSLGipioFZworNFWbEjZeM9yVhqIkFyz4AXwVB4lkdPTQTJrGNPngiSjZBJjq5HTjwzK4Lkk10jCb/GYnHopFyPXOCwRgWq6ZbvsCqL92IdKjLCTPIkZpNJKbmYql9yujpbCJy6RwKD5BSrKHHCFVkIYKS6ZKWzBeuRc4bIJIfCUkJXNeSOKGLaHl83yXLa5XILjwFqkt+JBCZVbmdSV5YWReqSg6Ew2rpHtdi0KYl65JUL3Lqf5C+M1CWfiDFIHvaOZycL09zb2chEyUVbzyiGRqMzuGLRnsmUnjU0Yuoeyy2IQXKKmUwmpRXcsdYBfTdGZfyyOTOCuWJih4t4DY4GMBZpucUgOXlKJlnn7haSJCldBZLJIgPjbeCA1C3e6x30oSvy+tWz1EIQdcntvaNRQ6imM6IqtxAlBjShX3Kk/7Ug3q/Kix1pWQtREskkD0Wm7lHuYpCcBiJIbusehcen/6XVUDiMviH5DJnBTu6odDsh5kMkMp5avdiTr5vkOW2iu4W+7wltPaNKnezKJINkV74N5ZET71QFyWLKHqDvoj1BdLgQk/dmI8ot8u1Www2Y0tOcigKl/ERd/gOoO1ukvh4ZGF+4B2BSVptyC4/QNBB1yRKA5nb9Sy76h/zKIgj2us0dVotZWXDX1R9/kBw1SIRXIJKmbgGnZ+cbdUCSaP9sNZFNTlmQHNne8mKHIU7W1DXRx2NYvMceyVOT65JLAExevJeu9m9CcUH01D3KXQyS02BBdZGSMThqgMV73YPqqWkMdnKJWPSSTCbZZjUrdfaUOFGTHJYkpW+uHkR/5DnlBVHBQaJEXXJbT2zlB/EIS5KSST5jYXyjs1Ml32FFVRyL9ziSenqi5KKj16PUA3t8QSWbm45Fe0B0Jpl1ybmNQXIaWC1m5YPjmAEW7/WwR3LOqnJHguQEapKVOvYSpyGCk0wnMsmAfm3gwup6ZFWv2mSIyXuSBJyMcxLdbNq6x0tDjFCPLIiSi5iCZJFJZpA8ifo1KE7e1Fe90pVJLonKJDNIzmUMktNkaaTkorl9CKGwvgsBuiOdLZx2KxeO5BiRiRkcDcRdC6sOkil56iBZr8V7radHMBpZJ7FCg1ILAJhfVQRxCqV1v+SDx8frkZOtn9ZSPIv3RJDPcovJ5pQXKL2jxcmb+qpXuoJk9dQ9llvkNgbJabKstgSAPJK09XTqWiPFQtSWVhQ7mBHMMdXu8QA33myyKNMpZz2yJoyQSY6uRy7R5Gc67VbUlMsLrJo1DpJFf+T5VYUoMlD7tIWqxXunTs+cTR6JtIArchpn+43CpOqXLNoSiuFHdptFGfKRDqLkgpnk3MYgOU2W1I63Rjqqcyu48YEQzAjmmuqy8dXhXXEEyWPBEAaG2RFFS2IsNQD4dOqVLC5pz63QNuhcJHoHazieeiwYxruR1mBGKrUA5Oy5MFO/ZEmSxsstmEme0srIyVpXnwf9w37lfaranZ/WpI6oz2cmObcxSE6TonybcqlI77rkbmUgBDOCuaak0AZ7nhycxZNJ7hn0QfRfYJCsDXUmWY82cOGwpPSjXbGgRNOfvWiOnBQ4PeCNmjCXjOb2QWWBoxFav6nlO6zKsJ6ZTgx8gRCCIflIYk3y1NQdVo6c6h/vbJGmRXtCSaGcSebCvdzGIDmNRF2ynkGyPxBSVgpz0V7uMZlMqCqV93s8QXL3ADuiaE3vIPlk17DyvCs1qkcWFqmGimhVl3wwMmXPajGjbm6JJj9TS6LkYqbFiuoTBna3mFpNWb7SPeedk/3jmeQ01SMLorSD5Ra5jUFyGi2LDBXpG/Kjbyj+scBaEOOoAWYEc5X4sIkvk8yOKFqz2yzKAjc9gmTRi9YEoE6jemRhbkUhLJFx0Vr1S34n0vpt2dxi2PIsszw6/US/5I7e0WkXYqqDZJZbTM1kMildLvYe6UYgMuVTnNyni8gkD3PqXk5jkJxGIpMM6JdNZkaQRJDc1eeNeYiFqGN3FdhgtxkvQMlEZpMJDjFQRON+wrE4fHIAgFxPW+DQNmDLs5oxv6oQAHBcg7pkj29MWQRotFILQbT5lCTgVNfUk/eGVSOpWW4xPdFpRb2gtSZN0/aE4kgmWQKn7uUyBslpVF2ar7wx6jVUpDsqI8ggOReJINk/Fop5UYo4ueKJlbbEQJF0Z5KDoTDejSwg1roeWVBP3kt2ouDhUwMQP8Joi/YE9eK96foli84WAAzVncNopuq0olcmGZBbZlJuYpCcRiaTCUsjJRfH9AqS++UguaTQhjwrM4K5qEpV29fZG1s7QvZITo3x0dTpDZJPdg4r/Xy16o880aJI7+DB0QD6h5Or6xRT9vLt1qgx0EbitKsW700XJEcyySYTlD68NFl1ab6SyQUAd5EdDlt6/17qdnMDSb5+KXMxSE4z0Qqu5fRI2gcISJKEA5Fm/LUVhWl9bjIO9QKYzn7vDI+USZKk1CSzHllbTps+QbKoRzabTKibV5KS5xDlB0DyJReHIov2Vi5ww2w2bm93pfVd59R12MORmuQCR56hfw+9yXXJ4ydv6V60BwDFqkzyADPJOYtBcpoti6zKDksSjrdr22h/Nic6h5XFWhtXVqb1uck4nHar0gNUPc1qOqO+ILyRPr4st9CWXplkMURkQXVRVJcNLdWUFSj169MFjbHoG/Ip71tnLDJmqYUgxlN39nqmTIKIhXtFXLQ3K/WI6iodgmS7euoeM8k5i0Fymi2sLlJWfR9N8+K9Vw52ApBbKK2vY5Ccy5TFe/2zB8mi1AIAKlluoSmlJjmNC/eCobDy3pOqemQAMJtNWBgpjWhOIiFw8MT4KGqjLtoTlMl7mHrxnii34KK92emdSQbGF++xDVzuYpCcZrY8i/JGms665FA4jNfeOQ0AWLusHPkO1sPlMpGZiSWTrA6SWZOsLT0yyc3tQ8pQDq37I08k+iWf6BxGOMHFe+9ESi3KXA7Dn6TNrypS2vpNVZcsyi0YJM+u0u3E+WfVYG5FIc5eoU9SRxkownKLnMVISQdLaovR1D6EpvZBhCUJ5jSM2nznRL/SxubcVVUpfz4yNpGZ6R70IhgKw2qZ/nxZBMkWsylqxTclT4+aZFGPbDGbotpSpoKYvOf1B3G63xt3RlCSJGXR3hkL3WkdS5wIsXivs8+Dk1OUmLDcInYmkwn/3/tX6roNykARllvkLGaSdbAs8sHk9YfQ3h1bd4FkvXKwCwBQ4LBi9eKytDwnGZcIViQJOD3L4j3R/q282MHFRhobbwGXvnILUY+8qMaV8o4BYiEbkNhQkbbuUQxFShSM2vptooXK4r3JmeQRj5yoKHSy/VsmEEkBLtzLXQySdSDawAHpqUv2B0J4491uAMDZK6tmzBpSbqguG8/odc0yeY/t31JHDBPxj4UQDifXSzgWY8EQjrXJwWoq65GFsmKHUlqQyELlQ6p65JUGr0cWFqoW76mvEIQlCSNe+WuWW2QG0eFieDSAUJhT93IRoyUdFBfalS4BxyIN/VPpzaPd8I/JmSqWWhAgZ4XFAtLZxlMzSE4dda9cbxpaQja1DSkjdlPVH1nNZDIpdcnHE+hwcSiS9Z5fWQhXhgzfWKBavNdyenzxntcfVOqyWW6RGUqipu6NzfxgykoMknWytLYEQHrGU796SC61KC92RGWxKXdZLWaUR4LejhmC5FA4jL4huR6vnO3fNKduv5aOumRRj2y1mNL2XiD6JZ/qGlEC9FgEQ2EcOTUAIHNKLYDpF++NcCR1xlGvwWCHi9zEIFknoi65e8CHwRQefEOjARxoli9ZblpVZfiFL5Q+NaIN3AxBct+QX8l+VXCQiOactvGpl+moSxb1yIvnFMOWl56JmyKTPBYMoy2ONRhNbYPKFTCjt35Tc9qtSjmTuj+06GwBAIXMJGeEqKl7DJJzEoNknahXlR9NYSu4197pUoKcTWdUp+x5KPNUlcpB70zlFmz/llpOR/oyyf6xEJoidcHqQQ2pJoJkIL6SCzFlz2oxKUOYMoUouTg5TSa5iJnkjBA1dW+Ei/dyEYNkncwpL1Autaay5EKUWiyoKsKc8oKUPQ9lHtHhYtgzhlHf1PV2DJJTy2lLX5B8rG0QocjiwJUL0peZdRXYUOaSg40TcXS4OHRSvgK2tLZYmdyXKcQQFfXivWHveJDF7haZwZ5nUT6n2QYuNzFI1onZZMKSWjnDkqoguavPo0y64oI9mkjds3a6bHLPoNz+rcBh5QCaFEhnTbIotcizmrF4TnrXJiiL9zomt0WbiscXxPF2+bGZVI8sLIz8vvLkPfn3ED2SLWaT0vqPjE+UXAyOMkjORQySdbQssnDmZOcwAmPa1yOKLLLJBGw8g0EyRVMHydPVJYtMcjmzyCmhDpZSPZpaLNpbWluMPGt63/pFkNzWParUGc/kyKl+pUwsE4Pk+VWFyuI9UXKhHknNtSGZQ+mVzHKLnMQgWUdidXkoLCXUaH8mkiThlYOdAIAzFrg5KY0mcRXY4Ihcxp4uk8z2b6mVZ7XAapEDplRmkn2BIE5EsrjprEcWRGY1LElKZnUmoh7ZabcqfYczicOmWrwX+X2VkdRctJdRlKl7XLiXkxgk62jxnGJlJLXWJRfNHUPKJLVNq7hgjyYzmUxKNrmzb+qpe2LaXkUx27+liiMNo6mPto7XI69IYz2ysLB6vC1aLCUXoh555QJ3xk55VCbvdURnkrloL7MUM5Oc0xgk68hus2BeVSEA4JjGHS5ejYyhtlnNWFdXoenPpuyhBMm9kzPJXn9QqaNkJjl1xECRVAbJoh7ZlmeO6jaRLlFt0Wa5atY35ENH5PWYSa3fJlpQLf+du/o8UccSeyRnlhJO3ctpDJJ1JuqSj7UNKjV4yQqGwnjtHTlIXrusPGpxEJGaCJJP93smvf7Y2SI9HJG65FT2SRb1yHVzS3QbSy+C8+ZZguR3IgE9AKzKwHpkYaFq8t6prmFVuQU7W2QSTt3LbQySdSb6JY/6glNm8xJx6EQfhiOX9s5lqQXNQGT3AsEw+oeia+5EqQUAZYw6aS/VmWSPL6hMftOj1EIQQfLpfu+0LQcB+f0LAMpcdlS6M/fkTL1470TnMEY88uV6ZpIzC6fu5TYGyTpTj4bVqi5ZlFoUOvOwalHmZmIo9arc07eB6xmUM8kmE1DqYpCcKuJKjy+QmiD53dYBiIsEK+brHyQD43W6E0mSpCzaW7mwNKO7QKgX7x3vGILHJ+9f1iRnFk7d015XVxcaGhqwceNGbNmyBTt27IDfP/PftrW1FfX19dizZ0/U7b/4xS+wZcsW1NfX46abboLXO34F9IUXXsDy5cuj/jU0NMS1rQySdVbqciiN9rWoS/YFgnjjaDcAYOPKSt0urVJmEFP3gMlBsii3KC1y8HWUQmLhnidFmWRRj+ywWbCgujAlzxGLeZWFsEQW4U1XctHWM4rBUTnjmsn1yIIouTh0oh+imIndLTILp+5pS5IkNDQ0wOv1YufOnbjrrruwe/du3H333TN+3/bt2+HxRH9G/eEPf8A999yDW265BQ8//DDeeustNDY2KvcfO3YMF154If76178q/2677ba4tpeffAawJJJNPqpBJvnNd3sQGJMXF7CrBc3GYbPCXSR/CEwOkiOdLVhqkVKi3MKXoppkpR55XgksZv3e8vOsZsytlIP06RbviSwyAKxckPlXwRZGFu+JRXsAM8mZRj11b5CZ5KQ1Nzdj37592LFjB5YtW4YNGzagoaEBzz333LTf85vf/Aajo6OTbn/kkUdw7bXX4sILL8SaNWvw/e9/H7/+9a+VbHJTUxPq6upQUVGh/HO54lu4zCDZAJbNLQEgr4Ie9iR3pip6I1eUOLBkTvpXsVPmEYv3Jg4UYY/k9BAL91KRSR72BNDSNQJA31ILYbEyeW+6IFmuR55bUYjigsxf4LZgih7PzCRnHvZK1k5FRQUeeOABlJeXR90+MjIy5eP7+/vR2NiIW265Jer2UCiEt99+Gxs2bFBuW7t2LcbGxnD48GEAcpC8cOHCpLbX0G0PzGbTjD0yLZFLwJYMvxS8Zkm5Mh2va8ALd4L1nyOeALxjISypLcb5a+cgLy/1o0+zZR9ksmT3weql5QgEw7DbLLBGJrGFJQmuQjsKnHlYvsCt3E5TS2YfzK0sxJLaYlgsJs3/zm29HiyOXKlav6JS9/24tq4cp07LH4YefxAuVSAcDIURCIaxpLYY555ZldC2Gu39aMncYiyZWwyoGseUlzh13w+pZLR9oIVVi8qQ78hDviMvI/adkfeBy+XCli1blK/D4TAeffRRbNq0acrH33777bjqqquwbNmyqNuHhobg9/tRWVmp3Ga1WlFSUoLOzk5IkoTjx4/jr3/9K+6//36EQiFcfvnlaGhogM0W+wm4SZI06juWApIkZfTCDSIiIiKa2h133IGdO3fiySefRF1dXdR9L7/8MrZt24bnnnsODocDy5cvxyOPPIJzzjkHHR0deO9734tdu3Zh3rx5yve8973vxT//8z9jw4YNuOiii3DVVVfh2muvRWtrK2677TZccskl+O53vxvz9hk6k9zXNzprJtnlcmJoyItQKLObfD/y+yM43jGEeVWF+KcPrEzoZ/z0NwfR0evBnPJ8XPehVRpv4dSyaR9kqmT3wdGWAfz3rqMAgC9duQpV7nyc7BzGL34nX7L6pw+uxLwK/RZ8ZYJk9sHbzb146k/NAIB/+eRazVqESZKEHz+5HwMjAZy5uBQfvWCJJj83GeGwhNt3voGxYBib19TgfevnKvftfrMNf97XDrPZhG9fsw62vMQyyUZ7P3rqz814u6kXAGC1mHHTZ9dldfLHiPsgWc//vQWvHOhEoTMP//LJtXpvzqz02Adud0Hc39PY2IiHH34Yd91116QA2efzYdu2bbj55pvhcEy+um63y2tpAoHoEtVAIACn04na2lrs2bMHxcXFMJlMWLlyJcLhML71rW/hxhtvhMUS25V2QwfJ4bCEcHj2RHcoFEYwmNkHY77dgqa2QZzsHManLlqGvDgv6XT0juKv+zsAyF0t0v33yIZ9kOkS3QcFDiuaIotGT7QPoazIgRMdQ8ptBXYr922MEtkHgUBI+VsPDvvh0KhM6nS/B3uPyJ1uzj+rxjD7MByW0NQ2CEeeGRecNUe5/S/72tHUNojl80pgNiGp7TXS+5HTZlH2r7vIjlBIQlT9RZYy0j5IVjAYRlPbIEwA/IGgrgtg42HkfXDrrbfiscceQ2NjIy677LJJ9+/fvx8tLS2TWrZdd911uPLKK7F9+3bY7Xb09PRgyRI5ARAMBjEwMICKCnnKcElJSdT3LlmyBH6/H4ODgygtjW1hcGbs6RwghooEQ2Gc7Jq6h+hMRG9ks8mEjSurNN02ym5lxQ6lNZfocCEW7dnzLFyNn2LqiZhaLt47eMKYk+sW1ciL2Y53DENU+3n9QTS3y4v5sqH1m9pC1eI9HkuZiVP3tHXPPffg8ccfx5133okrrrhiysesWbMGzz//PJ555hnlHwDcdttt+NrXvgaz2YzVq1dj7969yvfs27cPVqsVK1aswF/+8hecc845UX2T33nnHZSUlMQcIAMGzyTnkiVzimGCfBAeax2MGjIyG0mS8OohuavFGYvcWbEqnNLHYjaj0u1ER69Hmfqobv+WzZeGjUAdJPs0DJJFp4jq0nxDDYMRQ0U8/iBO93tRVZqPI6cGlLHoZ2TZAKT5lUUwmQBJYmeLTDVx6p5om0nxa2pqwr333outW7di/fr16O7uVu6rqKhAd3c3ioqK4HA4sGDBgknfX1VVhbKyMgDANddcg23btqGurg6VlZXYvn07Pv7xj8PpdKK+vh52ux3f/e538ZWvfAUtLS340Y9+hC984QtxbS+DZINw2q2orShEa/dI3JP3mtqHlKDm3DPYG5niV12aLwfJ/ZEgeZDt39LFaR8vr/Bo1Cs5HJaUISJGy8yqJ+8d7xhCVWm+EtA77daozGs2sNssmFNegLbuUSYwMlQxp+5p5sUXX0QoFMJ9992H++67L+q+I0eOYPPmzdixYweuvvrqWX/WFVdcgba2Nmzbtg2BQACXXnopvvWtbwEACgsL8fOf/xw//OEP8dGPfhQFBQX45Cc/ySA5ky2bWywHya0DcXX2EL2RbXlm1NeVz/JooslEr+TxTDKD5HTJV2eSNRpNfbJrGKORMchGKrUAgPJiBwqdeRjxjuF4xzA2rarGwUiQvGK+vgNPUuXjFy7Fi3tbcenZ8/XeFEpAScF45niQU/eSsnXrVmzdunXa+48cORLXfTP9vGXLluGhhx6KfyNVsu/dKIOJuuQhzxhOD3hnebQsGArj7++cBgCsq6tQRtwSxUMEyaO+IPqGfMoHQXmxcS7TZyv1MatVTfLB43LQaTaZsNwAQ0TUTCYTFip1yUPoH/ajI3JydobBAnqtrF5chq9/7Kwph4uQ8dltFuWKDzPJuYVBsoGo65CPtcZWcnHgeJ8y8nQTSy0oQVWRIBmQX1MCM8mpZzabYLfJH8Ba1SSL8oVFc4qQ7zDeifOiyLjmU13DONDcq9xutNIQIkHUJQ8wk5xTGCQbSHmxQ6l9irUu+dVIqUVRfh5WLeIHDCWmukwVJKuCFgbJ6eGMBMleDWqS/WMh5f3jjAXGzMwumiMHyYFgGC+83gpAbo9WrTpZIzISUU/OTHJuiTtI7urqQkNDAzZu3IgtW7Zgx44d8PtnftG0traivr4ee/bsSXhDc4HJZMKySDY5lkyy1x/Em0d7AADnrKzKylo+So8iZ55SG6tuHcZyi/QQHS68GtQkv9sygGBI7hSxyqCdItSL91q75THVqxaWspMKGVZJpKMFa5JzS1xRlSRJaGhogNfrxc6dO3HXXXdh9+7duPvuu2f8vu3bt8Pj8SSznTlj6dwSAEBbzyhGfTP3Y3zj3W6MRRqFb1rFUgtKnMlkUrLJ3sgl/+JCG2waDbagmSlBsgblFqLUwp5nweI5rlkerY/iAhtKXdFttFhqQUY2Xm7BTHIuiStIbm5uxr59+7Bjxw4sW7YMGzZsQENDA5577rlpv+c3v/kNRkdHk97QXKGuS25qG5rxsaKrRZXbqTToJ0pUlTv6UjdLLdJHBMlaLNw7eFy+ErB8fgmsFuNeXVJnkwFgZZYu2qPsUBIptxgaDSAUNuYUO9JeXCs6Kioq8MADD6C8PLrN2MjIyJSP7+/vR2NjIx588EF88IMfjHvjzGYTzObpL79ZIh8AFgN/EMRrca0LNqsZgWAYze2DWLe8YsrH9Q/78U6kD+p5q2uQp1PGLxv3QabRah/UVhREfV3ldsIa53j0XJXsPhCL6/yBUFJ/88ERv1K+sHpJmaH335LaYmVs9rzKQpRpUNrD9yP9Zes+KI28PiXI/czdRcZbECtk6z7QQ1x72eVyYcuWLcrX4XAYjz76KDZt2jTl42+//XZcddVVWLZsWUIbV1paEFONmsuVXRmvugVuHGjqxfHOEbjdBVM+5k/7OxAZUIXLzls07ePSJdv2QSZKdh8snV8KoEn5en5Nse6vq0yT6D4oKZI/gP3BcFJ/87eOj9eTn3tWraH335q6SvzqpWMAgHUrqjTdVr4f6S/b9sG8mvGrvCGTydDHlpBt+0APSZ0KNTY24tChQ3jyyScn3ffyyy9j7969M5ZizKavb3TWTLLL5cTQkBehUPZc/lhUVYQDTb04crIP3T3DU14y3fXaKQByNsZpMaG/X5+SlmzdB5lEq31QaDdP+Nqi2+sq0yS7D8Tb3IgnkNTf/LUD7QCAkkIbimxmQ++/iiIbChxWjPqCOHOhW5Nt5fuR/rJ1H1ghKf/f0j6I8kLjTk/UYx9kwklDIhIOkhsbG/Hwww/jrrvuQl1dXdR9Pp8P27Ztw8033wyHI/FLaOGwhHBYmvVxoVAYwWD2HIyLVe2RjrcPTarda+sZxcnOYQDApjOqDPG7Z9s+yETJ7oNyV/SxWuZycJ/GKdF94MgTLeCCCf/NJUnCgWZ50d7KBaUIhSQAs79/6iXPYsZ3P7cBQ54Alsxxafpa4/uR/rJtHxQ68pT/7x30ZcTvlm37QA8JFazceuuteOihh9DY2IjLLrts0v379+9HS0sLGhoaUF9fj/r6egDAddddh23btiW3xTlgySxDRURvZLPJhLNXVqZtuyi72fIsKFN1HODCvfQRC/d8/hDCUmKBbWefB/3D8sr7TOmZXlWaj2WRjj5ERsape7kp7kzyPffcg8cffxx33nknLr/88ikfs2bNGjz//PNRt1166aW47bbbcN555yW2pTmk0JmHOeUFaO8ZxdG2QVxy9jzlvrAk4dWDXQCAMxeXwpVv3Es+lHmqS/PRO+SH1WJWBttQ6olhIhLkxXsiaI7HIVV/65UGHSJClMmKC+zw+j2cupdD4nonbmpqwr333outW7di/fr16O7uVu6rqKhAd3c3ioqK4HA4sGDBgknfX1VVhbKysuS3OgcsrXWhvWcUx1oHIEmSsoDxWOsgeod8AIBNq6r03ETKQvOrinDwRD9qywtg5mCHtFEHxV5/MMEgWS61qC0vgLvIPsujiSheJYU2dPZ5MMhMcs6Iq9zixRdfRCgUwn333YfNmzdH/QOAzZs347e//W1KNjTXLK0tASDPiRdBMTBeamG3WVC/bOr2cESJuvyc+bhyyyL84wdW6L0pOWVikByvUDiMw6fkTPJKDuUgSgkxdY+Z5NwRV7pi69at2Lp167T3HzlyJKH7aLJlc6PrksuLnQiGwvj74dMAgPV1FbBzGhpprCjfhg+ft0jvzcg5UUFyIBT39x/vGIbXL3/fKg7lIEqJkgJO3cs17DRtUJVuJ4ry5dW0R9vkxXtvN/Vi1CdnmVhqQZQ9xIIgILFM8qHjcqmFxWxC3bwSrTaLiFRKIus0hjycupcrGCQblMlkUkZUN0U6XIgx1MUFNqxcwEuqRNki2XKLg5F65CVzXAnVMxPR7IoL5UyyJAFDo2M6bw2lA4NkA1saKblo6R5B35AP+471AgA2rqyCxcxdR5QtnLbEg2SvP4jm9iEAwBkstSBKmRJVx5/BUZZc5AJGWga2LLJ4T5KAX+0+hmBkcs65Z7LUgiib2PLMSjcRUVscqyMtAwhFhi6dsYhBMlGqlBSOd40ZGNZu8d4b73bjzv/Zh3dbBjT7maQNBskGtqC6EFaL/MH52jvygr2asnwsqCrSc7OISGMmk0mpS443kyxavzntFiyq4XsDUaqoe8cPaJRJ/uv+DvznU2/jwPE+PLbrqCY/k7TDINnA8qwWLKyOHkm96YwqpWcyEWUPUUscf5Ast35bMd/NMiyiFHLYrONT94aTD5J3v9mGB3/7jjI8/mTXMDp6R5P+uaQdvqMa3FJVKzgAOGdVtU5bQkSppATJgdiD5P5hP9p75A9V1iMTpV5xpA3c4Ghy5RbP/70Fv/yD3BrXVWBTyq32HOpKbgNJUwySDU50uADkgLmyxKnj1hBRqojR1PHUJItSCwA4g0NEiFJOLN5LJpP8f6+cwOMvyqUV7iI7/vXT65Tj99VDXZAkaaZvpzRikGxwS+cWK3XJ72EWmShrJVJuIUotSl12VJfmp2S7iGicWLyXyNQ9SZLw9J+b8es/NQMAyosd+NdPr0N1aT7OOUNekH+634vjHcPabTAlhUGywbnybfjylWfi4xcuxflr5+i9OUSUIvEGyZIk4dBJOZN8xoJSrlUgSgMlSI5z4Z4kSXhidxOeffkEAKDK7cS/fnodKiJXh9fVVSDPKodkrx7q1G6DKSkMkjNA/bIKXH7OfKVmiYiyz3hNcmzlFu09oxiMZLPOWMRSC6J0EB0uhkYDCIdjK4sISxL++4Wj+P1rpwAAc8oL8O1Pr0Opy6E8xmm3Yu3ScgByN6tYfzalFoNkIiIDcMTZAu5gpNQCkDPJRJR6Jeqpe57ZSy7CYQmP/P4wXnyjFQAwv7IQN1xTH9VzWdgUKbkYGg3gnZP9k+6n9GOQTERkAPmRTLIvxiBZLNqbV1kIV4FtlkcTkRbUU/cGRmYuuQiFw3jg/w7hz291AAAW1bjwrWvq4cqf+ng9c3GZ8j7AkgtjYJBMRGQAjsho6kAwrEzXnE4wFMaRUwMA2NWCKJ2ipu7NsHgvGArj/v89iFcPyi3dls0txjc/uRYFjrxpvyfPasaGFZUAgL1HuhEYi2/6JmmPQTIRkQGIDBIwe8lFU9sg/JEP0FXsj0yUNsUxZJLHgiHc+/QBvH6kGwCwcoEb3/j4WmXdwUxEyYUvEML+pl4NtpiSwSCZiMgA1B+gsy3eE63frBYTls0rSeVmEZGKw2aFI9LTfHCKTLJ/LIQf//pt7DvWAwBYvbgMX/uHNbBHvmc2dfNL4C6Ss9WvcrCI7hgkExEZgBh3CwBe38yZZFGPvLS2GPa82D58iUgb472SozPJXn8Q//HEWzh4XD4+65eV4/qrV8MWxzFqNplwzko5m7y/qQejvjGNtpoSwSCZiMgA1Jlk3wyjqT2+IJo7hgBwFDWRHqaauufxBXHnr/bhcGStwMaVlfjylWcqvY/jIQaLBEMS9kZKNkgfDJKJiAzAoQqSPTPUJB8+1Q8xtXbVIgbJROk2PlBELrcY8Y6h8fE30dQmn7yet7oaWz+0ClZLYiHW/KpC1JTJEzT3sORCVwySiYgMQL1wz+efviZZlFoUOKxYUFWU8u0iomhi8d7AiB9DowH86L/fwMlOeZT0e+tr8Y8fWAmzOfHhXyaTSVnAd/hkP/qH45vuR9phkExEZAAO1cKemTLJYojIigXupD6IiSgxIpM8NBrAHf/9Blq7RwEAl2yYh89eWqfJdFxRciEBeO0dZpP1wiCZiMgArBYzbJH6xelqknsHfejq8wBg6zcivYhMsiQBHb3y8XjFuQvwyfcthUmDABkAKt35WDLHBQBKr2VKPwbJREQGIeqSp8ski1ILgENEiPTinjBS+soti3D1+Ys1C5AFkU0+2TWMjt5RTX82xYZBMhGRQTiV0dRT1yQfOimXWpQXO1Dpzk/bdhHRuJryAqVrxccvXIoPn7dI8wAZAM5eWaWUbnABnz5mH/9CRERp4YzUJU81cS8sSUomma3fiPTjyrfhe9duwFgwjEU1rpQ9T3GBDWcsdOPA8T68erALH9mcmmCcpsdMMhGRQThnKLdoPT2CYY88WICt34j0NbeiMKUBsiBKLk4PeHG8Yzjlz0fRGCQTERlEvlJuMTlIFqOoTQBWLmA9MlEuWFdXoZR2vHqoU+etyT0MkomIDMIRGU3tmaIm+WCk1GJ+dREKnXlp3S4i0ofTbsXapeUAgNfeOY1QOKzzFuUWBslERAahLNyb0AJuLBjC0ZYBAOxqQZRrxGCRodEADp8c0HdjcgyDZCIig3Da5CB54sK9Y62DCATlDBL7IxPlltVLylDgkN8bXj3Ikot0YpBMRGQQIpPs9YcgSZJyu2j9lmc1Y9ncYl22jYj0YbWYsX55JQBg77vdCIxNP7aetMUgmYjIIJyRmuSwJCEwNl57ePC4XI9cN7cYeVbLlN9LRNlLlFz4AiHsb+rVeWtyB4NkIiKDEJlkAPBG6pJHvGM42Sm3fmJ/ZKLcVDe/BO4iedLfKyy5SBsGyUREBhEVJEfqkg+f7IcovGCQTJSbzCYTzlkpZ5Pfbu7FqG9M5y3KDQySiYgMIjpIlusOxZS9Qmce5lUV6rJdRKQ/MVgkGJKw90i3zluTGxgkExEZhBhLDYxnkg8qo6jdMHMkLVHOml9ViJqyfADscpEuDJKJiAxiYrnF6QEvugd8AFhqQZTrTCaTsoDvyKkB9A/7dd6i7McgmYjIICYGyaLUAuAQESIaL7mQAOw51KXvxuQABslERAZht1kgCirkIFnuj1zldqK82KnfhhGRIVS687FkjgsAg+R0YJBMRGQQZpMJjkg2edQXxDtKPTJLLYhIJrLJJ7uG0dE7qvPWZDcGyUREBiIGihxpGcCoT168xyCZiISzV1Ypi3hfPchscioxSCYiMhBRl3y0ZQAAYDIBKxeU6LdBRGQoxQU2ZY3CnkNdUSPsSVsMkomIDMRpk4Nk8bG3qMaFfEeefhtERIYjSi5OD3hxvGNY563JXgySiYgMRN3hAmBXCyKabF1dBfKscgjHnsmpwyCZiMhARE2ysIr1yEQ0gdNuxdql5QCA1w6fRigc1nmLshODZCIiA1Fnkm15ZiyeU6zj1hCRUYnBIkOjAbxzsl/nrclODJKJiAxE1CQDwPJ5buWSKhGR2uolZShwyO8Xe9jlIiX47ktEZCDqcotVrEcmomlYLWasX14JANj7bjcCYyGdtyj7MEgmIjIQdbkF+yMT0UxEyYUvEMJbTb06b032YZBMRGQgKxeWwmm3YPm8EtRWFOi9OURkYHXzS+AusgNgl4tUsM7+ECIiSpfa8gLc/dXNsFrMMEWmahERTcVsMuGclVX4/Wun8HZzL0Z9YygutOu9WVmDmWQiIoPJs1oYIBNRTDatkksugiEJe49067w12YVBMhEREVGGmldZiJqyfAAsudAag2QiIiKiDGUymZQFfEdODaBvyKfzFmUPBslEREREGeycVdUAAAnAnkPG7pnc1dWFhoYGbNy4EVu2bMGOHTvg9/tn/J7W1lbU19djz549Ubf/4he/wJYtW1BfX4+bbroJXq9Xuc/v9+Omm27Chg0bsHnzZjz44INxbyuDZCIiIqIMVlnixJI5LgDAKweMW3IhSRIaGhrg9Xqxc+dO3HXXXdi9ezfuvvvuGb9v+/bt8Hg8Ubf94Q9/wD333INbbrkFDz/8MN566y00NjYq9//oRz/CgQMH8PDDD+Pmm2/GPffcg9///vdxbS+DZCIiIqIMd06k5OJE5zBauoZ13pqpNTc3Y9++fdixYweWLVuGDRs2oKGhAc8999y03/Ob3/wGo6Ojk25/5JFHcO211+LCCy/EmjVr8P3vfx+//vWv4fV64fF48MQTT+A73/kOVq1ahUsuuQRf+MIXsHPnzri2l0EyERERUYY7e2UVzJGuOH96s1XnrZlaRUUFHnjgAZSXl0fdPjIyMuXj+/v70djYiFtuuSXq9lAohLfffhsbNmxQblu7di3GxsZw+PBhHD58GMFgEPX19cr969evx1tvvYVwOBzz9hq6T7LZbILZPH0bJIvFHPVfSj/uA/1xH+iP+8AYuB/0x32gn7JiBy4+ey6a2oZwom0IZrMJVqux9oPL5cKWLVuUr8PhMB599FFs2rRpysfffvvtuOqqq7Bs2bKo24eGhuD3+1FZWancZrVaUVJSgs7OTpjNZrjdbthsNuX+8vJy+P1+DAwMoLQ0tmmmhg6SS0sLYuoV6nI507A1NBPuA/1xH+iP+8AYuB/0x32gj69+Yp3emxCXxsZGHDp0CE8++eSk+15++WXs3bt3ylIMn0/u4KEOgsXXgUAAkiRNeR8ABAKBmLfP0EFyX9/orJlkl8uJoSEvQqHY0+ekHe4D/XEf6I/7wBi4H/THfaCvcFjCX9/uQLk7H6sXlaZtH7jdBXF/T2NjIx5++GHcddddqKuri7rP5/Nh27ZtuPnmm+FwOCZ9r90uTxWcGPAGAgE4nU6EQqEp7wMw5c+bjqGD5HBYQjgszfq4UCiMYJAHo564D/THfaA/7gNj4H7QH/eBfs4/aw7c7gL0948adh/ceuuteOyxx9DY2IjLLrts0v379+9HS0sLGhoaom6/7rrrcOWVV2L79u2w2+3o6enBkiVLAADBYBADAwOoqKiAJEno7+9HMBiE1SqHut3d3XA4HHC5XDFvp6GDZCIiIiLKHvfccw8ef/xx3Hnnnbj88sunfMyaNWvw/PPPR9126aWX4rbbbsN5550Hs9mM1atXY+/evTjnnHMAAPv27YPVasWKFSsAyDXK+/btUxb37d27F6tXr4bZHHudNoNkIiIiIkq5pqYm3Hvvvdi6dSvWr1+P7u5u5b6Kigp0d3ejqKgIDocDCxYsmPT9VVVVKCsrAwBcc8012LZtG+rq6lBZWYnt27fj4x//OJxOuR5eZJx/+MMf4vTp03jwwQexY8eOuLaXQTIRERERpdyLL76IUCiE++67D/fdd1/UfUeOHMHmzZuxY8cOXH311bP+rCuuuAJtbW3Ytm0bAoEALr30UnzrW99S7r/xxhuxfft2XHvttSgsLMRXv/pVXHrppXFtr0mSpNmLfnXS3T1zM2yr1Wz4uptsx32gP+4D/XEfGAP3g/64D/Snxz6oqChKy/Okm7Ea6BERERERGQCDZCIiIiKiCRgkExERERFNwCCZiIiIiGgCBslERERERBMwSCYiIiIimiCuILmrqwsNDQ3YuHEjtmzZgh07dsDv90/52D/+8Y/4yEc+gvr6enzoQx/Ciy++qMkGExERERGlWsxBsiRJaGhogNfrxc6dO3HXXXdh9+7duPvuuyc99vDhw7j++uvx0Y9+FM888ww++clP4mtf+xoOHz6s5bYTEREREaVEzBP3mpubsW/fPvztb39DeXk5AKChoQF33HEHvv3tb0c99rnnnsOmTZvwuc99DgCwYMECvPTSS/jd736nzNQmIiIiIjKqmIPkiooKPPDAA0qALIyMjEx67FVXXYWxsbFJtw8PzzxBj4iIiIjICGIOkl0uF7Zs2aJ8HQ6H8eijj2LTpk2THrtkyZKor48ePYpXXnkFn/zkJ+PaOLPZBLPZNO39Fos56r+UftwH+uM+0B/3gTFwP+iP+0B/3AfaMUmSJCXyjXfccQd27tyJJ598EnV1ddM+rq+vD9dccw3Ky8vxyCOPwGyOfadJkgSTafogmYiIiIgoFWLOJKs1Njbi4Ycfxl133TVjgNzT04N//Md/hCRJ+PGPfxxXgAwAfX2js2aSXS4nhoa8CIXCcf1s0gb3gf64D/THfWAM3A/64z7Qnx77wO0uSMvzpFvcmeRbb70Vjz32GBobG3HFFVdM+7iuri5l4d4jjzyCqqqq5LaUiIiIiChN4sok33PPPXj88cdx55134vLLL5/2cR6PB1/4whdgNpvxyCOPoKKiIukNJSIiIiJKl5iD5KamJtx7773YunUr1q9fj+7ubuW+iooKdHd3o6ioCA6HA/fffz9OnTqFX/7ylwCgPNbhcKCoqEjjX4GIiIiISFsxl1v89Kc/xb//+79Ped+RI0ewfPly7NixA1dffTUuv/xyHD9+fNLjrrrqKtx+++3JbTERERERUYol3N2CiIiIiChbsYkeEREREdEEDJKJiIiIiCZgkExERERENAGDZCIiIiKiCRgkExERERFNYPgg2e/346abbsKGDRuwefNmPPjgg9M+9tChQ/jYxz6Gs846Cx/96Edx4MCBNG5p9opnH3z5y1/G8uXLo/7t3r07jVub3QKBAD74wQ9iz5490z6Gx0FqxbIPeBykRldXFxoaGrBx40Zs2bIFO3bsgN/vn/KxPA5SJ579wGMhNU6ePInPf/7zqK+vx3vf+1488MAD0z6Wx0Li4pq4p4cf/ehHOHDgAB5++GG0t7fj29/+NubMmTNp4p/H48HWrVvxoQ99CLfffjsee+wxfPGLX8QLL7yA/Px8nbY+O8S6DwB56ExjYyPOPfdc5bbi4uJ0bm7W8vv9+Jd/+RccPXp02sfwOEitWPYBwOMgFSRJQkNDA1wuF3bu3InBwUHcdNNNMJvN+Pa3vx31WB4HqRPPfgB4LKRCOBzG1q1bsXr1ajz99NM4efIkvvGNb6Cqqgof+tCHoh7LYyFJkoGNjo5Kq1evll599VXltv/8z/+UPvOZz0x67BNPPCFddNFFUjgcliRJksLhsHTJJZdIv/71r9O2vdkonn3g9/ullStXSs3NzencxJxw9OhR6cMf/rD0oQ99SKqrq4vaH2o8DlIn1n3A4yA1jh07JtXV1Und3d3Kbc8++6y0efPmSY/lcZA68ewHHgup0dXVJX3ta1+ThoeHldu+8pWvSDfffPOkx/JYSI6hyy0OHz6MYDCI+vp65bb169fjrbfeQjgcjnrsW2+9hfXr18NkMgEATCYT1q1bh3379qVzk7NOPPugubkZJpMJ8+bNS/dmZr3XXnsN55xzDv7nf/5nxsfxOEidWPcBj4PUqKiowAMPPIDy8vKo20dGRiY9lsdB6sSzH3gspEZlZSXuvvtuFBYWQpIk7N27F3//+9+xcePGSY/lsZAcQ5dbdHd3w+12w2azKbeVl5fD7/djYGAApaWlUY9dunRp1PeXlZXNelmUZhbPPmhubkZhYSFuuOEGvPbaa6iursZXv/pVXHDBBXpsela55pprYnocj4PUiXUf8DhIDZfLhS1btihfh8NhPProo9i0adOkx/I4SJ149gOPhdS76KKL0N7ejgsvvBCXXXbZpPt5LCTH0Jlkr9cbFZwBUL4OBAIxPXbi4yg+8eyD5uZm+Hw+bN68GQ888AAuuOACfPnLX8bbb7+dtu3NdTwO9MfjID0aGxtx6NAh/PM///Ok+3gcpM9M+4HHQur9+Mc/xn/913/hnXfewY4dOybdz2MhOYbOJNvt9kk7UnztcDhieuzEx1F84tkH/+///T989rOfVRZlrFixAgcPHsSvfvUrrF69Oj0bnON4HOiPx0HqNTY24uGHH8Zdd92Furq6SffzOEiP2fYDj4XUE39Hv9+Pb37zm7jhhhuigmIeC8kxdCa5qqoK/f39CAaDym3d3d1wOBxwuVyTHtvT0xN1W09PDyorK9Oyrdkqnn1gNpsnrVpevHgxurq60rKtxOPACHgcpNatt96Khx56CI2NjVNeXgZ4HKRDLPuBx0Jq9PT0YNeuXVG3LV26FGNjY5Nqw3ksJMfQQfLKlSthtVqjCsz37t2L1atXw2yO3vSzzjoLb775JiRJAiC3qXnjjTdw1llnpXOTs048++Bf//VfceONN0bddvjwYSxevDgdm0rgcWAEPA5S55577sHjjz+OO++8E1dcccW0j+NxkFqx7gceC6nR2tqK66+/Pupk48CBAygtLY1aJwTwWEiWoYNkp9OJK6+8Etu3b8f+/fuxa9cuPPjgg/jc5z4HQM5o+nw+AMDll1+OoaEh/OAHP8CxY8fwgx/8AF6vF+9///v1/BUyXjz74KKLLsKzzz6LZ555BidPnsQ999yDvXv34jOf+Yyev0LW43GgPx4HqdfU1IR7770X1113HdavX4/u7m7lH8DjIF3i2Q88FlJj9erVWLVqFW666SYcO3YMf/rTn9DY2IgvfelLAHgsaEq35nMx8ng80g033CCtXbtW2rx5s/TQQw8p99XV1UX1+nvrrbekK6+8Ulq9erX0D//wD9LBgwd12OLsE88++NWvfiVdeuml0plnnildddVV0muvvabDFme3iT16eRyk32z7gMeB9u6//36prq5uyn+SxOMgXeLdDzwWUqOzs1P6yle+Iq1bt04677zzpPvuu0/phcxjQTsmSYrk4ImIiIiICIDByy2IiIiIiPTAIJmIiIiIaAIGyUREREREEzBIJiIiIiKagEEyEREREdEEDJKJiIiIiCZgkExERERENAGDZCIiIiKiCRgkExEZ3EUXXYSf/OQnem8GEVFOYZBMRERERDQBg2QiIiIiogkYJBMRzWJ4eBjf+973sGnTJqxfvx6f+9zn8PbbbwMAfvKTn+BTn/oU/vM//xPnnHMONmzYgBtvvBEjIyPK9w8MDOD73/8+LrjgAqxZswaf/OQnsWfPnqjn+Mtf/oJPfOITOOuss3D++efjrrvuQigUUu7v7u7G9ddfj7Vr1+Kcc87Bjh07ou4nIiJtMUgmIpqBJEm47rrr0NLSgvvvvx+/+tWvsHbtWnzqU5/CoUOHAABvv/02/vrXv+LBBx/Ef/7nf+Lvf/87vv71rwMAQqEQ/umf/gmvv/46Ghsb8dRTT6Gurg6f//znsX//fgDAm2++ia1bt2L9+vV46qmncNttt+Hxxx/Hvffeq2zHk08+ibPPPhvPPvssvvWtb+EXv/gFnn766bT/PYiIcoVJkiRJ740gIjKqV155Bf/4j/+IV199FSUlJcrtn/nMZzB37lzU1tbipz/9KXbt2oWqqioAwJ///Gdcd911+N3vfoeWlhZs3boVzz77LOrq6gDIgffVV1+N+fPn4z/+4z/wjW98A+3t7Xj88ceVn/+HP/wBvb29uOaaa3DRRRdhzZo1uPvuu5X7P/zhD2PdunXYvn17Ov4MREQ5x6r3BhARGdnBgwchSRIuvPDCqNsDgQD8fj9qa2uxcOFCJUAGgHXr1gEA3n33XbS0tKCoqOj/b+f+WVqH4jiMPyXqJLgJ4qAQUSIOHbqICm6C0NGpoIuL4KAovoAuUpz8uxREd6mvQEEQEVHsHJDqa6goKtg7XFpornbxioPPB0I4OTmHnCV8SX5JIyADpFIpMpkM5+fnjfPGxsaa5p+ammpq9/f3N7W7urp4eXn58vokSR8zJEtSC+/v73R2dlIqlf7p6+jo4OjoiPb29qbj9VrhIAj47GVdrVajre3vLbi+byUIgg/nkCR9D2uSJamFwcFBHh8feXt7o6+vr7EVi0VOTk4AqFQqVKvVxpjb21sAhoeHGRoaolqtEsdxo79Wq3Fzc8PAwAAAYRg2PgSsOzw8ZGZm5ruXJ0n6hCFZklqYmJggiiKWl5e5vLzk4eGB9fV1SqUSYRgC8PT0xNraGnEcc3FxQT6fZ3p6mt7eXsbHx4miiJWVFa6urri7uyOfzxPHMXNzcwDMz89TLpfZ3Nzk/v6es7Mz9vb2mJyc/MGVS9LvZrmFJLUQBAH7+/tsbGywtLTE8/MzYRiys7PD6Ogo19fX9PT0EEURuVyOIAjIZrOsrq42jS8UCiwuLvL6+srIyAgHBwek02kAoihid3eXra0tisUi3d3dzM7OsrCw8IMrl6Tfzb9bSNIXbG9vc3x8zOnp6U9fiiTpP7LcQpIkSUowJEuSJEkJlltIkiRJCT5JliRJkhIMyZIkSVKCIVmSJElKMCRLkiRJCYZkSZIkKcGQLEmSJCUYkiVJkqQEQ7IkSZKU8AeAlU4eeVNpqgAAAABJRU5ErkJggg==", + "application/vnd.jupyter.widget-view+json": { + "model_id": "46ca413f7112493bbeb2be6403b68e9f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/30 [00:00" ] @@ -972,32 +1283,11 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 21, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqYAAAHmCAYAAAChy0xuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABM70lEQVR4nO3deXydZYH+/+ss2fd9T9qmSZq2SZvu0NLSIpuAIoogAo6izDC4IQ4MfH8z4jaM8v06bigKVhlBWWTHBa2UpbSlNE2XpEuWpmn2fV/OyTnn+f1xkkhFoGlz8jxJPu/Xi5ftyWlypbdPe/V+7vt+bIZhGAIAAABMZjc7AAAAACBRTAEAAGARFFMAAABYAsUUAAAAlkAxBQAAgCVQTAEAAGAJFFMAAABYAsUUAAAAluA0O8DZam/vf8+P2+02xcdHqKtrUD4fzxIwA2NgPsbAfIyBNTAO5mMMzGfGGCQlRZ3W+2b9jKndbpPNZpPdbjM7ypzFGJiPMTAfY2ANjIP5GAPzWXkMZn0xBQAAwMxAMQUAAIAlUEwBAABgCRRTAAAAWALFFAAAAJZAMQUAAIAlUEwBAABgCRRTAAAAWALFFAAAAJZg+iNJR0dHdeedd6qlpUVhYWG67777FB8fb3YsAAAATDPTZ0z/8Ic/KCUlRb/5zW902WWX6cEHHzQ7EgAAAExg+ozphz/8YV122WWSpJaWFsXExJicCAAAAGaYtmL6+OOP69e//vUpr/3iF79QSkqKnE6nbr75Zh06dEi//OUvpysSAAAALGTaiuk111yja6655l0//vOf/1x1dXW6+eab9dJLL01XLAAAAFiE6WtMH3/8cT3yyCOSpPDwcNntpkcCAACACc64Bbrdbl1++eV68803J15zuVy6++67tWrVKm3YsEFbt259389z6aWXaseOHbr++uv1pS99Sd/85jfPNBIAAABmsDO6le9yuXT77berqqrqlNe/+93vqry8XA8//LCampp05513Kj09XZdccsm7fq7o6Gg98MADZxJDkmS322S329714w6H/ZT/na3+uLtONY29mpcWrbzMGM1Pi1ZwkMPsWJLmzhhYGWNgPsbAGhgH8zEG5rPyGEy6mFZXV+v222+XYRinvD40NKQnn3xSDz74oJYsWaIlS5aoqqpKjz766HsW07MVHx8hm+3di+m46OiwgGUwW1V9t367zf+PhD1H2iRJDrtNuZkxWjQvXoVj/yXEmPt7MJvHYKZgDMzHGFgD42A+xsB8VhyDSRfTPXv2aO3atbrtttu0fPnyidePHj0qj8ejkpKSiddWrlypBx54QD6fL2BrR7u6Bt93xjQ6Okx9fcPyen0ByWC2Z7dXS5LsNpsMGTIMyeszVHmyR5Une/T8a8clSQnRocrLilFeZqzyMmOUlRIpxzSs6Z0LY2B1jIH5GANrYBzMxxiYz4wxiIuLOK33TbqYXnfddf/w9fb2dsXFxSk4OHjitcTERLlcLvX09ATsaU4+nyGfz3jf93m9Pnk8s+8CGBwZ1e6KFknShuJUXXtBnmqb+lTd2Kvqxj7VNPZqyOWRJHX2jaizYkS7K1olScFBdi1Ii9bCzBgtzIjRgvQYRYYFBSzrbB2DmYQxMB9jYA2Mg/kYA/NZcQym7Lio4eHhU0qppImfu93uqfoy+Ds7D7XIPfZ/qs0lmQoNdk7cupckn2GouXNINY29qm7oVVVjr1q7hiRJ7lGfjp7s0dGTPROfLy0hXAsz/EU1LytWKXFhp7VUAgAA4GxNWTENCQl5RwEd/3loaOhUfRm8jWEY2l7WKElakB6tnNSod7zHbrMpIzFCGYkR2rgsXZLUP+RWTeP4rGqvapv7NDpWbps7h9TcOaTXDzZLkqLDgyZu/edlxSp7mm7/AwCAuWfKimlKSoq6u7vl8XjkdPo/bXt7u0JDQxUdHT1VXwZvc7SuWy1js5+bSzJO+9dFhQdreV6iluclSpI8Xp/q2wZU3dA7UVa7+12SpL6hUZVWtqu0sl2SFBLk0IJ0/87//KxYLUiPVmiw6U+2BQAAs8CUNYrCwkI5nU7t379fq1atkiSVlpaqqKiIQ/MDZHy2NCLUqdWLks/48zgdds1Pi9b8tGhduDpLktTZO6Kqhh5VNfSqsqFHje2DkiTXqFdH6rp1pK5bkn9GNic18m+zqpmxio4IftevBQAA8G6mrJiGhYXpyiuv1D333KP/+q//Ultbm7Zu3ap77713qr4E3qZnwKWyqg5J0vqitCk/szQhJlQJMalatyRVkn+TVfVYSa1q6NWJ5j55vIZ8hqHa5n7VNvfrz2/VS5JS4sP9M6qZscrLilF64untxAMAAHPblN6Dveuuu3TPPffoU5/6lCIjI/WFL3xBF1100VR+CYx57UCTvGOnEUzmNv6ZiggN0rKFiVq20H/7f9TjVW1z/8SsalVDr4bHdv+3dg2ptWtIO8bWqcZEBmvpgkQtSItSXkaM0pMiZGdDFQAA+Ds24+9Pyp9h2tv73/PjTqddcXER6u4etNyRCGfK6/Ppjp/uUne/S4vnxemr15a8/y8KMJ/PUGPHoKoaelRZ7y+r4+tU/15EqFP5WbEqyIpVQXacspIj3/MsWpy92XgdzDSMgTUwDuZjDMxnxhgkJb1zg/Y/wq6VGehgdedE6ZuO2dLTYbfblJUcqazkSG1ZkSnDMNTZN6Kqev9mqqrGXjW0DUiSBkc8KqvqmFiKEBbiUF6mv6jmZ8cqJyVKTgs+Jg0AAAQWxXQGGt/0FBsZPLGz3mpsNpsSY8KUGBOm85anKy4uQnUN3Tpc26VjJ3t0rL5bDWMbqoZdXh2s6dTBmk5J/p3/CzOilZ8dp4KsWM1Pi1aQk6IKAMBsRzGdYdq6h1Re2yVJ2rgsfUadKRodEaxVi5K1auwEgYHhUVXV9+hYfY+OnezRybZ+GYZ/53/FiW5VnPDv/A9y2pWbHu2//Z8dp9z06Cnf7AUAAMxHMZ1hXtnfJMl/TNOm5da4jX+mIsOCVJKfpJL8JEnS0IhHVQ3+olpZ36MTzf3yGYZGPW97QtUbJ+R02LQgPUaFOXFalB2rBekxzKgCADALUExnkFGPd2Kn+/K8RMVFhZicaGqFhzpP2fk/4vaourF37NZ/j2qb+uT1GfJ4DVWOldfnJAU77VqYOV5U4zQvLWpGzSQDAAA/iukMsvdouwaGRyVZZ9NTIIUGO7V0foKWzk+QJLlHvapp6tOxk906WtetmrGi6vb4dPhEtw6P3foPDXYoPytWi7LjVJjDrn8AAGYKiukMMr7pKTkuTIXz4kxOM/2CgxwqzPGXTZ0nudxeVTf26khdt46e7FZtc58MQxpxn7qZavx4qsKcOC3KiVNGYoRsnKMKAIDlUExniPq2AVU39kqSzl+ewQH1kkKCHVoyP15L5sdLkoZdHlXW90wU1frWARl65/FU0eFBKhibTV2UE6eUuDCKKgAAFkAxnSHGZ0udDrs2FKeZnMaawkJOXaM6MDyqYyd7dHSsqDZ2+I+n6hsa1VtH2/TW0TZJUlxUiBbPi9OSefEqnBevmIhg074HAADmMorpDDDs8mhXRYskaU1hsiLDgkxONDNEhgVpZUGSVhb4d/33Drp17GS3f0a1rlut3cOSpO5+l9441KI3Dvl/jzOTIrR4XrwWz4tXQVasQoI5mgoAgOlAMZ0Bdle0yOX2Spobm54CJSYiWGsKU7SmMEWS1NU3oiN1/qJacaJLvQNuSVJD+6Aa2gf157fq5bDbtDAjRovnxWnx/HjNS2XHPwAAgUIxtTjDMPTy2G387ORILUiPNjnR7BEfHar1RWlaX5QmwzDU1Dmkwye6dLi2S0fre+Rye+X1Gf4HANT36JnXaxUW4lRhTpy/qM6LZ30qAABTiGJqcVUNvWoce3Tn+SsyKEEBYrPZlJEYoYzECF24Kkser0+1zX2qqO3S4bpuHW/sk88wNOzyaF9lu/ZVtkuSEqJDVDgv3l9Uc+IVzfpUAADOGMXU4l4Zmy0NDXZo3eIUk9PMHU6HXXmZscrLjNWV5/nX+R472aPDJ7pUcaJLzZ1DkqTOPpd2HGyeePBBVnKkls6P19IFCcrLjJHTwW1/AABOF8XUwvqG3Np7zL9z/NylqQoNZrjMEhbi1PK8RC3P8+/47+53+W/7n+jS4RPd6h30r0+tbxtQfduA/vjmSYWMnbu6dEG8ls6PV3JcuJnfAgAAlkfTsbAdB5vl8RqS2PRkNXFRIaeuT+0YVMWJblXUdunYyW65PT65Rr3aX92h/dX+81OT48ImZlMXZcfyDw0AAP4OfzNalM8wJm7j52fGKCMp0uREeDc2m00ZSZHKSIrURauzNOrxqrK+V4eOd6qitmvi/NS27mG93N2ol/c1ymG3KT8rdmw2NUGZSTyNCgAAiqlFlR/vUkfviCT/pifMHEHOU59I1dU3ovLaLpUf71TFiW4Nuzzy+oyJo6qe3F6j2MhgLZkfr6IFCVo8L56zagEAcxLF1KLGZ0ujw4O0Mj/Z5DQ4G/HRodq4LF0bl6XL6/Optqlfh453qry2Syea+2RI6hlwTxzyb5M0Pz1aS8eK6vy0aNntzKYCAGY/iqkFdfaO6ECNf13iecvSFeRkZ/ds4bDbtTAzRgszY/SRjQvUP+TW4RPdKh8rqr2DbhmSjjf16XhTn55/44SiwoNUtCBBxbkJWjo/XuGhzKYCAGYniqkFvXqgUYYh2SRtWpZudhwEUFR4sNYuTtHaxSkyDEP1bQMTt/2rGnrl9RnqHxrVzvIW7Sxvkd1mU15mjIoXJqg4N1HpCeGsTQUAzBoUU4vxeH167YD/TMyi3AQlxoaZnAjTxWazKTslStkpUfrguhwNuzw6fKJbB2o6dKimU72DbvmMvz2J6sntNUqMCVVxrr+kLsqOVXCQw+xvAwCAM0YxtZh9le3qGzsTkyOi5rawEKdWFiRpZUGSfIahk639OljdqQM1nRNrUzt6R/TyPv9O/2CnXYU5cSpemKhluQmKjw41+1sAAGBSKKYWM77pKSE6VEULEkxOA6uw22yalxqteanR+tCG+eobdOvQcX9Jrajt1LDLK7fHpwM1/td+LSkzKULFuYkqzk1QQU6s2d8CAADvi2JqIU0dgzp6skeSdH5JOjux8a6iI4InDvj3eH2qbujVwZpOHajpmHhcakP7oBraB/WH3XWKCAvSqkUpWjIvVkvmxSsshEsfAGA9/O1kIeOzpQ67TecVs+kJp8fpsGtRTpwW5cTp41sWqq1nWIdqOnWgukNHT/bI4/VpcHhUr5Y16NWyBjnsNi3KiVNJXqKWL0zklj8AwDIophbhcnv1RnmLJGllQZKiI4JNToSZKjk2TBeszNQFKzPlcnt1uK5L5ce7dKCmU119I/L6DFXUdqmitkuP/LlSOalRKslLVEleEk+gAgCYimJqEW8eadWwyyOJTU+YOiHBDpXkJWl1YYpiYsJVdqRZe4+2qayqQ43t/kel1rX0q66lX8++XqvEmFAtHyup+Vkxctg5QxcAMH0ophZgGIa27/Pfxs9IjFB+Vqy5gTAr2e02LUiPUXZylK7amKu27iHtr+pQWVWHKht6ZBj+Xf7b9jZo294GRYQ6VZyboJK8JC2Zz7pUAEDg8TeNBdQ296uutV+SdH5JBrdSMS2S48J10ZpsXbQmWwPDozpQ3aH9VR0qr+2Sa9SrwRGPdlW0aldFq5wOmwpz4lWSl6hlCxMVFxVidnwAwCxEMbWA7WUNkqTgILvOWZJqchrMRZFhQRO7/Ec9Xh0+0a2yqg7tr+5Q36BbHq+hQ8c7deh4p/TSMc1Pi9aK/EStKkhWSny42fEBALMExdRkA8Oj2nOkTZK0bnGqwkMZEpgryOnQsoX+mVGfYai2qU9lVR0qq2qfOIqqtrlPtc19eurV48pMitDKgmStKkhSeiKbpwAAZ44WZLKdh5o16vFJYtMTrMdusyk3I0a5GTH62Pm5au0aUllVh/ZVtaumoVeGxs9LrdVzO2qVGh+ulQVJWlWQrOyUSEoqAGBSKKYmMgxD2/c3SZIWpEcrJzXK5ETAe0uJD9cla7N1ydps9Qy4tK+yXaXH2nX0ZLcMQ2rpGtLvd9Xp97vqlBgTOlFS56dHy05JBQC8D4qpiY7Udau1y39rlNlSzDSxkSHasiJTW1Zkqn/IrbKqDpUea9fhE13y+gx19I7opT31emlPveKiQrQiP0mrCpKUlxnLU80AAP8QxdRE28ee9BQR6tTqRckmpwHOXFR4sDYuS9fGZekaGhnV/mp/ST10vEser0/d/S79tbRBfy1tUHR4kEry/TOpBdmxcjo4KxUA4EcxNUl3v0tllR2SpPVFaQoOcpicCJga4aFBOndpms5dmqYRt0cHazq191i7DtV0yjXqVd/QqF7d36RX9zcpItSp5Xn+3f1L5sdTUgFgjqOYmmRnebN8hiGJ2/iYvUKDnVpTmKI1hSlyj3pVXtul0mNt2l/dqWGXR4MjHr1xqEVvHGpRRKhTKwuStKYwRQXZsTx1CgDmIIqpSY7WdUvyb3riHEjMBcFBDq3IT9KK/CR5vD4dPtGt0mP+x6MODI9qcMSj1w4067UDzYoOD9KqRclaU5iihZkxbJwCgDmCYmoCr8+n6sY+SeLxo5iTnA67inMTVJyboBu8Ph2t69abR1q1r7Jdwy7/7f6X9zXq5X2NiosK0ZpCf0mdlxrFEVQAMItRTE1wsnVArlGvJIop4HTYtXRBgpYuSNCNF/tUfrxTbx5p1f7qDrlH/Runxnf3J8eGac1if0nNTIo0OzoAYIpRTE1QWd8jSbJJysuMMTULYCVBTrtK8pNUkp8kl9urAzUd2nOkTQdrOuXx+tTWM6wXd9bpxZ11ykiMmJhJZTkMAMwOFFMTjBfTjKRIRYQGmRsGsKiQYMfExqmhEY/Kqtq150ibKmq75DMMNXYM6pnXa/XM67XKSY3S2sIUrV6UrISYULOjAwDOEMV0mvkMQ1UNvZKk/CxmS4HTER7q1PqiNK0vSlP/kFulle3ac7hVx072yJBU19KvupZ+PbG9WgszYrRuib/QRobxDz8AmEkoptOsuWNQA8OjklhfCpyJqPBgnb88Q+cvz1B3v0t7j7Vpz5FW1YxtKKxu7FV1Y69+u61KxbkJOndpqopzExXk5PgpALA6iuk0qxybLZUopsDZiosK0YWrsnThqix19AzrraNt2lXRqob2AXl9hsqqOlRW1eF/ulphis5dkqrcjGh29gOARVFMp9n4+tLkuDDFRoaYGwaYRRJjw3Tpuhxdui5H9W0D2lXeol2HW9Q74NbgiEevlDXqlbJGJceG6ZylqTpnSYqS49g0BQBWQjGdRoZhTBTT/MxYU7MAs1lWcqSytizUx87P1eG6Lu0qb1FpZbvco/6d/c/tqNVzO2q1MCNG5y5N1erCZDYiAoAFUEynUUfviLr7XZKkPDY+AQFnt9u0dH6Cls5P0A1uj0qPtWtXRYuOnOiWob+tR/3Ntkoty03UOUtTVZybIKeD9agAYAaK6TQany2VpALWlwLTKjT4bzv7u/pG9ObhVu2saFFj+6A8XkOlle0qrWxXZFiQVhcm69wlqVqQznpUAJhOFNNpVNXQI0mKiQxWUmyYuWGAOSw+OlSXrsvRJWuzVd82oJ3lLdp9uFV9g24NDI9q+75Gbd/XqJS4MJ27NFXri9IUH835qAAQaBTTaXSs3r8jvyArllkYwAJsNpuyU6KUnRKlqzfn6vCJbu0qb9G+yna5PT61dg/rmddr9eyOWi2dn6CNy9K0bGEit/oBIEAoptOkd9Ct1q4hSVIeG58Ay3HY7SpakKCiBQkadvnXo+4sb9bRkz0yDOnQ8U4dOt6pqPAgrV+apvOWpSktIcLs2AAwq1BMp0kV60uBGSMsxKkNxWnaUJymtp5h7TjYpDcOtai736X+oVH9ac9J/WnPSS3MiNF5y9K0elGyQoP54xQAzhZ/kk6T8Y1PEaFOpScxywLMFMmxYbpqY66u3LBA5bWdeu1Asw5Ud8jrM962q79KawuTdV5xOhumAOAsUEynyXgxzcuMlZ2/tIAZx263qTg3UcW5ieoddGtXeYteO9Cklq4hudxevXagWa8daFZGYoTOK07TOUtTFRUebHZsAJhRKKbTYGjEo/q2AUmcXwrMBjERwbpkbbYuXpOl6sZevX6gWXuOtso96lNjx6Aee7laT75So5L8JG0sTtPiefFmRwaAGYFiOg2qG3tljP04n/WlwKxhs9mUlxmrvMxYfeIDedpzpFWvH2zW8aY+eX2G9h5t096jbYqPDtHGZem6YuNCBXHDBADeFcV0Gozfxg8OsisnJcrcMAACIizEqU3LM7RpeYYa2ge042Czdpa3aGB4VF19Lj37uv8xqMW5CdpckqGl8xNkt9NSAeDtKKbToHLsYP3c9BjOPwTmgMykSF17QZ4+uilX+6s79PqBJlXUdskwpAPVnTpQ3anEmFBtLsnQhuI01qICwBiKaYC5R72qbeqTxG18YK4Jctq1elGyVi9KVs+gW28eadOfdp1Q76BbHb0jevKVGj3zeq1WL0rWlhUZ7OgHMOdRTAOsttm/1kyS8jPZ+ATMVYkxobr+0kJdvDpTew636uV9jaqs75HH69OuihbtqmhRdkqktqzI1NrFKQoJcpgdGQCmHcU0wI6NrS912G1akEExBeY6p8OuNYUpWlOYoob2AW0va9TO8ha53F6dbB3Qr/54VE+8XK31RWnavCJDqfHhZkcGgGlDMQ2w8Sc+zUuNYgYEwCkykyJ1w0UF+timXO2qaNH2fY1q7BjUkMujv+yt11/21mvxvDhtWZGpZQsT5LCzRh3A7EYxDSCvz6fqRtaXAnhvYSFObVmRqc0lGaqs79H2skaVHmuX12fo8IluHT7RrbioEJ2/PF0bl6UrJjLE7MgAEBAU0wA62Tog16hXkpRHMQXwPmw2mwqy41SQHafeAZdeO9CkV/Y3qbvfpe5+l555vVbPv3FCKwuStLkkQ/lZsWyWAjCrUEwDaPz8UpukPDY+AZiEmMgQXbF+vj54To72V3Vqe1mDDp/oltdnaM+RNu050qbMpEhdtDpLaxenKMjJbX4AMx/FNIDGi2lGUqQiQoPMDQNgRnLY7VpZkKSVBUlq7hzUK2VN2nGoWcMujxraB7T1D0f0u1drdMGKDJ1fksGZqABmNIppgPgMQ1UNvZKk/CxmSwGcvbSECH3iA3m6auMC7T7coj+/Va/mziH1Dbr1zOu1enFXndYvTdWFq7OUlhBhdlwAmDSKaYA0dwxqYHhUEhufAEytkGCHNi3P0HnL0lV+vEt/fuukDp/o1qjHp1f2+9elFucm6OLVWVqUE8c6VAAzBsU0QCrHZksliimAwLDbbCrOTVBxboLq2wb057dOandFq7w+QwdrOnWwplNZyX9bh8ojkQFYHX9KBcj4+tLkuDDFcrQLgADLSo7UTZct1n3/eq4uP3eeIsP869rr2wb0i98f0b/9dKde3Hli4k4OAFgRM6YBYBjGRDHNz4w1NQuAuSU2MkRXbVygy87J0a5y/zrUlq4h9Q649fRrx/XizhNaX5SmC1dn8VQpAJZDMQ2Ajt4Rdfe7JHEbH4A5QoIcOr8kQxuXp+tQTaf+/Fa9jtR1y+3xaXtZo7aXNWr5wkRdtDpLBdmchwrAGiimATA+WyqxIx+Auew2m5YtTNSyhYk62dqvP79VrzcP+9eh7q/u0P7qDmWnROri1dlaXZjMOlQApuJPoACoauiRJMVGBispNszcMAAwJjslSp+9fLG+e8u5uuycHEWE+ucmTrYO6MEXD+vun+/W9rJGjXq8JicFMFcxYxoAx+rHzy/l9hgA64mLCtFHN+Xq8nPmaWd5s/78Vr1au4fV0TuiX790TM+/UatL1mTr/OUZCgl2mB0XwBxCMZ1ivYNutXYNSWJ9KQBrCwl2aPOKTG1anqHSyna9uPOE6tsG1Dvg1uMvV+v3u+p04apMXbAyU+E8vQ7ANKCYTrGqt68vZUc+gBnAbrdp9aJkrSpI0sGaTr2484Rqmvo0MDyqZ16v1Z/2nNSWFZm6cHWWonnkKYAAophOsfGNTxGhTqUn8UhAADOHbWyjVHFugo7WdevFXXU6UtetYZdXv99Vp7/srdemZRm6ZG224qI4nxnA1KOYTrHxYpqXGSs760sBzEA2m02F8+JVOC9eNY29enHnCR2o6ZR71Ke/7K3X9rIGbShK06XrctjgCWBKUUyn0NCIR/VtA5KkPI6JAjAL5GbE6EtXL9PJ1n69uKtOpUfb5PEaemV/k1470Ky1i1N02Tk5Sk/kDhGAs0cxnULVjb0yxn7MxicAs0l2SpT+9cqlau4c1B921WlXRat8hqFdFS3aXdGiFQVJuvycecpJjTI7KoAZjGI6hcZv4wcH2ZWTwh/OAGaftIQI3XT5Yn14w3z98c2Tev1gszxen0qPtav0WLuKcxN0+TnztDCTu0YAJo9iOoUqxw7Wz02P4ekpAGa1xNgw3XBxga5YP08v7Tmp7WWNco/6dLCmUwdrOlWYE6erNi1QbjoFFcDpoz1NEfeoV7VNfZK4jQ9g7oiNDNE1W/J03y3n6vJz5yksxD/fcaSuW9/+31L96KmDahhbew8A74cZ0ylS29wnr8+/wpRiCmCuiQoP1lUbF+iSNdnaVlqvl/ac1LDLq7KqDu2v6tDaJSn68Ib5SokLNzsqAAujmE6RY2PrSx12mxakR5sbBgBMEh7q1IfWz9eWFZn64+46/bW0QW6PT7srWvXWkTadV5ymK9bP5xxUAP8QxXSKjD/xaV5qlEKCeLY0gLktMixIV29eqAtXZ+nFnSf06v4meX3+Y6Z2HGrRlhUZ+uA5OTxJCsApWGM6Bbw+n6obWV8KAH8vNjJE119UoP+6eZ3WL02VzSZ5vD79+a163fnALj37+nENjXjMjgnAIiimU+Bk64Bco15JUh7FFADeISk2TDddvljfuGmtVhYkSZJcbq+ef+OE7nxgp/74Zt3En6MA5i6K6RQYP7/UJimPs/sA4F1lJEbo1o8U6T//aZWWLoiXJA2OePTk9hr9+8926eV9DfJ4fSanBGAWiukUGC+mGUmRiggNMjcMAMwA81Kj9ZWPL9ed15VM/IO+d8CtR/5cqbt/vltvHGqWz2e8z2cBMNtQTM+SzzBU1dArSSrgNj4ATEpBdpz+/ZMr9OWrlyk7JVKS1NE7ol/8/oj+4xdvqvRYmwyDggrMFezKP0vNHYMaGB6VJOVlcRsfACbLZrOpODdBSxfEq/RYu5557bhauobU3Dmk+58p17zUKF17QR6bS4E5gGJ6lirHZkslduQDwNmw22xavShZK/ITtau8Vc/tOK7OPpdOtPTrvx/dp1WLkvXx83OVGBtmdlQAAUIxPUvj60uT48IUG8mB0QBwthx2uzYUp2nt4hS9sr9Rz++o1eCIR3uPtml/VYcuWZulD67LUWgwf4UBsw1rTM+CYRgTxZTZUgCYWkFOuy5claV7//kcXbAyU3abTR6vTy/urNNd4xukWH8KzCoU07PQ0Tui7n6XJCk/M9bcMAAwS0WGBemTF+brGzetmThiqnfArV/8/oi+9fBeVTX0mBsQwJShmJ6F8dlSScpn4xMABFR6YoS+8vHl+vLVxUqND5cknWjp172P7NMDz5Wrs3fE5IQAzhYLdM7C+L/SYyODlcRifACYFsW5iVo8L17b9zXquR21GnJ5tOfI+PrTbF26NkchwQ6zYwI4A8yYnoVj9f4d+flZsbLZbCanAYC5w+mw68LVWbr3n9dp84oM2WyS2+PT82+c0N0P7tau8hbWnwIzEMX0DPUOutXaNSSJjU8AYJao8GDdcFGBvv6ZNVoyL06S1N3v0oMvHtZ//bpUNY297/MZAFgJxfQMVb19fSkbnwDAVJlJkfrKNcv1xY8WKyXOv7TqeFOfvv3rUv38hQp19bH+FJgJWGN6hsY3PkWEOpWeFGFuGACAbDabluclaumCeP21tEHPv3FCwy6Pdle0at+xdl26LkeXr59ndkwA74EZ0zM0XkzzMmNlZ30pAFiG02HXxWuyde8/r9P5JX9bf/rcjlrd+dOdeuNgkwzWnwKWRDE9A0MjHtW3DUhifSkAWFV0eLBuvLhA93x6jQpz/OtPu/pc+u+H39L3nzzA7X3AgiimZ6C6sVfj/9bO4/xSALC0rORIffXa5frCVUVKjAmVJJVVduj/PPSmtu2tl8/H7ClgFRTTMzB+Gz84yK6clChzwwAA3pfNZlNJfpL+65/X6cpNubLZJJfbq99sq9K3f106cRcMgLkopmegcuxg/dz0GDkd/BYCwEwRGuzUTR9aqns+s0bZKZGSpNrmPn3jV2/pd6/UyD3qNTkhMLfRqibJPepVbVOfJNaXAsBMNT8tWv/xqVX6+OaFCg6yy+sz9IfddfrPX+xRxYkus+MBcxbFdJJqm/vkHVuPRDEFgJnLYbfrkrXZ+uZNa7V0frwkqa1nWP/vsf36xYuH1T/kNjkhMPdQTCfp2Nj6UofdpgXp0eaGAQCctaTYMN328WW6+YrFigoPkiS9Ud6i//Pgm9pV3sLRUsA0ophO0vgTn+alRSkkyGFuGADAlLDZbFq3JFXf/tw6bShKkyQNDI/qwRcP63uP71dbz7DJCYG5gWI6CV6fT9WNY+tLeQwpAMw6kWFB+sxlhfq3a5creezRphUnuvWfD72pP+6uk8frMzkhMLtRTCfhZOuAXGM7NllfCgCzV+G8eH3jM2t0+bk5cthtcnt8evKVGn3z4b2qbe4zOx4wa1FMJ2H8/FKbpLxMDtYHgNksOMihqzbm6mufXq3csT0F9W0D+tb/7tVvt1VpxO0xOSEw+1BMJ2G8mGYmRyo8NMjcMACAaZGZFKm7rl+pT16Yr9BghwxD+sveev3HQ2/qQHWH2fGAWYViOgl1rf2SWF8KAHON3W7TBSsz9a3PrlVJXqIkqbPPpR/87qAe/tNRudwczA9MBYrpJCzLTVRsZLA2laSbHQUAYIL46FB94aPFuvUjRYqJDJYkvbq/Sff86i3WngJTgGI6CTdcXKDvfX6DMpMizY4CADDRyoIkffOmtVqZnyRJau0a0n/9ulS/33VCPh/nngJnimIKAMAZiAwL0r9+ZKk+fekihQQ55PUZeurV4/rub8vU0cu5p8CZoJgCAHCGbDabzluWrns+s3riaYCV9T362ta3tPtwi8npgJmHYgoAwFlKiQvXv39yha44d55sNmnY5dHPnz+sn79QoaERjpUCThfFFACAKeB02PWRjQv0759cocSYUEnS7opWfW3rnonjBgG8N4opAABTKC8zVl//zBqduzRVktTZN6Lv/Gafnnq1hkeaAu+DYgoAwBQLC3Hqs5cv1r98eInCQ5wyDOn3u+p07yOlaukaMjseYFkUUwAAAmRNYYq+cdMaLcqOlSTVNvfrnl/u0av7G2UYHCsF/D2KKQAAARQfHaqvXluiq8/PlcNuk3vUp4f/dEw/fvqQ+ofcZscDLIViCgBAgNntNl26Lkf/342rlBofLkkqq+rQf/5ij8qPd5qcDrAOiikAANMkJzVKX/v0am0uyZAk9Q669b0nDug3f6nUqMdrcjrAfBRTAACmUUiQQzdcXKAvfqxYUeFBkqRtpQ36xq/2qrFj0OR0gLkopgAAmGD5wkR946a1Ks5NkCQ1dgzqW/+7V6XH2k1OBpiHYgoAgEliIoL1pY8V65MX5stht8nl9ur+Zw7pmdeOy8eufcxBFFMAAExks9l0wcpM3XFdiWIigiVJL+w8oR/97iCPM8WcQzEFAMAC8jJj9Z//tFoL0qMlSQdqOvWt/92r5k7WnWLuoJgCAGARcVEhuvO6FdpQnCZJauka0jcf3quyKtadYm6gmAIAYCFBTrs+fekiXX+Rf93piNurHz11SM/tqGXdKWY9iikAABZjs9m0ZUWm/u0TJYoeO1LquR21uv/pQxp2se4UsxfFFAAAi8rP8q87nZcaJcn/tKhv/e9etXQNmZwMCAyKKQAAFhYfHaq7rl+h9UtTJUnNnUP65sNv6UB1h8nJgKlHMQUAwOKCnA595rJCXfeBPNltNg27vPrh7w7qhTdYd4rZhWIKAMAMYLPZ9IFVWfrqtcsVGRYkQ9Izr9fqp8+Us+4UswbFFACAGWRRTpy+9k+rlZPiX3daWtmub/+6VK3drDvFzEcxBQBghkmI8a87PWdJiiSpqWNQ3/zVXh063mlyMuDsUEwBAJiBgoMc+uzli3XtBf51p0Muj77/xAH9ftcJGaw7xQxFMQUAYIay2Wy6aHWWbr9m2cS606dePa6fPlehETfrTjHzUEwBAJjhCufF6z8/tUrZyZGSpL1H23TvI/vUM+AyORkwORRTAABmgcTYMN11w0qtXexfd1rfNqB7HylVW8+wycmA00cxBQBglggJcujmKxbrinPnSZLae0Z07yOlamgbMDcYcJoopgAAzCI2m00f2bhA116QJ0nqHXDrvx/dp+rGXpOTAe+PYgoAwCx00eos3XRZ4cSO/f/7WJnKazlOCtZGMQUAYJZaX5SmW69aKqfDLveoTz948qDeOtpmdizgXVFMAQCYxUrykvSVjy9TaLBDXp+hB54t1yv7G82OBfxDFFMAAGa5RTlxuuO6komzTv/3T8c4iB+WRDEFAGAOmJcarbuuX6H46BBJ/oP4n9xeQzmFpVBMAQCYI9ISInT39SuVlhAuSfrTnpP65R+PyuvzmZwM8KOYAgAwh8RHh+rOT65QTmqUJGnHwWb99NkKjXq8JicDKKYAAMw50eHBuuMTJVqUHStJ2lfZru8/eVDDLo+5wTDnUUwBAJiDwkKcuu3jy1SSlyhJOlLXrf/7WJn6h9wmJ8NcRjEFAGCOCnI69K8fWar1RamSpNrmfv33o/vU1TdicjLMVRRTAADmMIfdrk9/sFAXrc6SJDV3DuneR0rV0jVkcjLMRRRTAADmOLvNpmu2LNRVGxdIkjr7XLr3kVLVtfSbnAxzDcUUAADIZrPp8nPn6YaL8mWT1D80qu/+dp+Onew2OxrmEIopAACYsHlFpm7+0BI57DYNu7z63hMHtL+6w+xYmCMopgAA4BRrF6foix8rVrDTrlGPTz9+6pBKj7WZHQtzAMUUAAC8Q9GCBN1+7XKFhzjlMwz97PkKHanjtj4Ci2IKAAD+obzMWN1+7XKFBDnk8Rr60VMH2RCFgKKYAgCAdzU/LVq3XrVUDrtNI26v/ueJ/Wrt5igpBAbFFAAAvKel8xN00+WFsknqGxrV9x7fr94Bl9mxMAtRTAEAwPtatzhVn/hAniSpvWdE33vigIZGPCanwmxDMQUAAKflA6uydPm58yRJ9W0D+uFTBzXq8ZobCrMKxRQAAJy2j5w3X5uWp0uSKut79MBzFfL6fCanwmxBMQUAAKfNZrPphosKtDI/SZJUVtWhX790TIZhmJwMswHFFAAATIrdbtPNH1qsRdmxkqTXDjTr6deOmxsKswLFFAAATFqQ06EvfLRY2SmRkqTf76rTX96qNzkVZjqKKQAAOCNhIU7d9vHlSo4NkyT99q9V2lXRYnIqzGQUUwAAcMZiIoL1lWuXKyYiWJK09fdHdOh4p8mpMFNRTAEAwFlJjg3TbR9fprAQh7w+Q/c/c0g1Tb1mx8IMRDEFAABnLTslSl/8aLGcDrvcoz59/4kDauoYNDsWZhiKKQAAmBIF2XH6lw8vkc0mDY549L0n9qurb8TsWJhBKKYAAGDKrMhP0qcuWSRJ6upz6f89vl8Dw6Mmp8JMQTEFAABTauOydH100wJJUnPnkL7/5AG53Dy6FO+PYgoAAKbcB9fl6AOrMiVJx5v6dP+zh+Tx8uhSvDeKKQAAmHI2m03XXpCndYtTJEnlx7u09Q9H5OPRpXgPFFMAABAQdptNn7msUEsXxEuSdle06rd/qZRBOcW7oJgCAICAcTrsuvXKIi1Ij5YkvbSnXk9trzY5FayKYgoAAAIqJNihL1+9TGkJ4ZKk//3DYVXUdpmcClZEMQUAAAEXGRak2z6+TBFhQTIM6YFny9U76DY7FiyGYgoAAKZFYkyYbr5isSSpd9CtB1+okM/HelP8DcUUAABMm5L8JF25KVeSdPhEt36/64S5gWApFFMAADCtbvzg4onNUM/uqNWxk90mJ4JVUEwBAMC0CnLadetVRQoPccowpJ89X6G+IdabgmIKAABMkBQbps9cVihJ6hlw66EXDnP4PiimAADAHCvyk/SBlf7HlpbXdumPu+tMTgSzUUwBAIBprt68UDmpUZKkZ16rVVVDj7mBYCqKKQAAME2Q065bPrxEYSEO+QxDDzxXoYHhUbNjwSQUUwAAYKrkuHD906X+9abd/S794sXDMlhvOidRTAEAgOlWL0rW5pIMSdKBmk69tKfe5EQwA8UUAABYwrUXLFR2cqQk6alXa1TT2GtyIkw3iikAALCEIKdDt1y5VCHBDnl9/vWmgyOsN51LKKYAAMAyUuLD9alLCiRJnX0j2vr7I6w3nUMopgAAwFLWLU7VxmXpkqSyqg5tK20wORGmC8UUAABYznUfyFNmUoQk6YmXq1Xb3GdyIkwHiikAALCc4KCx9aZB4+tNyzU04jE7FgKMYgoAACwpLSFCN1ycL0lq7xnRr/7IetPZjmIKAAAs69yladpQlCZJ2nusXdvLGk1OhECimAIAAEv75IX5Sk/0rzd97K9VqmvpNzkRAoViCgAALC0k2KFbPrxEwU67PF5DP32uXMMu1pvORhRTAABgeRlJkfrkhf71pm3dw3r4T0dZbzoLUUwBAMCMsKE4TecsSZEk7TnSptcONJmcCFONYgoAAGYEm82mGy4uUGp8uCTpN9uqVN82YHIqTCWKKQAAmDFCg5265cqlCnLaNerx6afPlmvEzXrT2YJiCgAAZpSs5Eh94gN5kqSWriG9uLPO5ESYKhRTAAAw42xalq7i3ARJ0p/fOqm27iGTE2EqUEwBAMCMY7PZdO0FeXLYbfJ4DT3+crXZkTAFKKYAAGBGSo0P14WrsiRJZVUdqjjRZXIinC2KKQAAmLEuP3eeosODJEmPbauS1+czORHOBsUUAADMWOGhTl21KVeS1NgxqFfKONt0JqOYAgCAGW1DUZpyUqIkSc++flwDw6MmJ8KZopgCAIAZzW63TRwfNTji0XOv15qcCGeKYgoAAGa8/KxYrSlMliRtL2tUQztPhJqJKKYAAGBWuPr8hQp22uUzDP12W5UMwzA7EiaJYgoAAGaFhJhQXbouR5J0pK5b+6s6TE6EyaKYAgCAWeOStdmKjw6RJD32cpVGPRwfNZNQTAEAwKwREuTQxzcvlCS194zoL3vrTU6EyaCYAgCAWWX1omTlZcZIkl7YeUI9Ay6TE+F0UUwBAMCsYrPZdN0H8mWT5HJ79dQrNWZHwmmimAIAgFknJzVK5y1LkyS9Ud6i4019JifC6aCYAgCAWekjG3MVFuKQJP1mW6V8HB9leRRTAAAwK8VEBOuKc+dLko439enNilaTE+H9UEwBAMCs9YFVmUqJD5ckPflKtUbcHpMT4b1QTAEAwKzldNj1iQv8x0f1DLj1h911JifCe6GYAgCAWa04N1FFCxIkSX96s17tPcMmJ8K7oZgCAIBZ79oLFspht8nj9emJ7dVmx8G7oJgCAIBZLy0hQheszJQklR5r15G6bpMT4R+hmAIAgDnhQ+vnKTIsSJL0221V8vp8JifC36OYAgCAOSE8NEhXbVogSWpoH9BrB5pNToS/RzEFAABzxsbidGUlR0qSnnntuAZHRk1OhLejmAIAgDnDbrfpug/kSZIGhkf13I5akxPh7SimAABgTinIjtOqRcmSpJdLG9XYMWhyIoyjmAIAgDnn4+fnKshpl88w9Nhfq2QYhtmRIIopAACYgxJjw3TJmmxJUkVtlw5Ud5qcCBLFFAAAzFEfXJejuKgQSdJjL1dp1MPxUWajmAIAgDkpJNihq8/PlSS1dQ9rW2m9yYlAMQUAAHPW2sUpWpgRI0l64Y0T6h1wmZxobqOYAgCAOctms+kTY8dHjbi9euZ1jo8yE8UUAADMafPTorV+aaokaWd5iwaGOXTfLBRTAAAw5108tkPf4/Vpx0EeVWoWiikAAJjzMpMjtTDTv9b0lbJG+TjX1BQUUwAAAEmbSzIkSW09wzp8osvkNHOTZYppbW2tVqxYYXYMAAAwR60qSFZkWJAkafu+RpPTzE2WKKbDw8P6zne+o5CQELOjAACAOSrIadd5xWmSpP3VHerqGzE50dxjiWL67W9/W7feeqvCwsLMjgIAAOawTSUZskkyDOm1A01mx5lznNP1hR5//HH9+te/PuW1X/ziF3rllVe0aNEiFRUVTVcUAACAfyg5NkxLFsSr/HiXXj3QpMvPnSenwxLzeHPCtBXTa665Rtdcc807Xn/++edlt9v10ksvqb29XZ/97Gf10EMPTVcsAACAU2wuyVD58S71Dri1v6pDqxYlmx1pzpi2YvpuHn300Ykfb9myhVIKAABMtSw3UfHRIerqc2l7WSPFdBqd8dy02+3W5ZdfrjfffHPiNZfLpbvvvlurVq3Shg0btHXr1ikJCQAAMF3sdps2LUuXJB2p61ZL15DJieaOM5oxdblcuv3221VVVXXK69/97ndVXl6uhx9+WE1NTbrzzjuVnp6uSy655LQ+78svvzzpLHa7TXa77V0/7hhbF+JgfYhpGAPzMQbmYwysgXEw30wZg80rM/X8Gyfk9Rl69UCTPnlhvtmRpoyVx2DSxbS6ulq33367jL97IsLQ0JCefPJJPfjgg1qyZImWLFmiqqoqPfroo6ddTM9EfHyEbLZ3L6bjoqPZ8W82xsB8jIH5GANrYBzMZ/UxiIuL0LqiNL1xoElvHGzW5z5SrJAgh9mxppQVx2DSxXTPnj1au3atbrvtNi1fvnzi9aNHj8rj8aikpGTitZUrV+qBBx6Qz+eT3R6YVt7VNfi+M6bR0WHq6xuW1+sLSAa8N8bAfIyB+RgDa2AczDeTxuC8olS9caBJA8OjeumN4zpv7Pb+TGfGGMTFRZzW+yZdTK+77rp/+Hp7e7vi4uIUHBw88VpiYqJcLpd6enoUHx8/2S91Wnw+Qz7f+z/P1uv1yeOx9gUw2zEG5mMMzMcYWAPjYL6ZMAZ5GTFKSwhXc+eQtu1t0DlLUs2ONKWsOAZTNo05PDx8SimVNPFzt9s9VV8GAABgWthsNp2/PEOSVNvcp7qWfpMTzX5TVkxDQkLeUUDHfx4aGjpVXwYAAGDarC9KVbDTX5e2lzWYnGb2m7JimpKSou7ubnk8nonX2tvbFRoaqujo6Kn6MgAAANMmPDRIaxanSJJ2H27V0IjnfX4FzsaUFdPCwkI5nU7t379/4rXS0lIVFRUFbOMTAABAoG0u8d/Od4/6tLO82eQ0s9uUNcawsDBdeeWVuueee3Tw4EFt27ZNW7du1Y033jhVXwIAAGDazU+L1rzUKEnS9rLGdxyZiakzpVOZd911l5YsWaJPfepT+vrXv64vfOELuuiii6bySwAAAEy78VnT5s4hHTvZY26YWeyMnvw07tixY6f8PCwsTN/5znf0ne9856xCAQAAWMmaxSl6/OVqDbk82l7WqEU5cWZHmpVY/AkAAPA+QoIcOrfIf47pvsp29Q64TE40O1FMAQAATsP47Xyvz9BrB9kEFQgUUwAAgNOQlhChRdmxkqTX9jee1pMnMTkUUwAAgNO0eUWmJKmzz6WDNZ0mp5l9KKYAAACnqSQvUTER/keuby9rNDnN7EMxBQAAOE1Oh10bl6VLksqPd6q9Z9jkRLMLxRQAAGASNi1Pl80mGZJe2c+s6VSimAIAAExCfHSoli9MlCS9fqBZox6fyYlmD4opAADAJI0fHTUwPKrSY20mp5k9KKYAAACTtHh+vJJjwySxCWoqUUwBAAAmyW6zaVOJfxNUVUOvGtoGTE40O1BMAQAAzsCGojQ5Hf4qtZ1NUFOCYgoAAHAGosKDtXpRkiRpV3mLRtwekxPNfBRTAACAM7S5xP8kqBG3V7srWk1OM/NRTAEAAM5Qbka0MpMiJfk3QRmGYXKimY1iCgAAcIZsNps2r/AfHVXfNqCapj6TE81sFFMAAICzsG5xikKCHZKk7fsaTE4zs1FMAQAAzkJYiFPnLkmVJL11tE39Q26TE81cFFMAAICzNP4kKI/X0I5DzSanmbkopgAAAGcpMzlSCzNjJEmvljXJxyaoM0IxBQAAmALjs6ZtPcM6XNtlcpqZiWIKAAAwBVYVJCsyLEiS/+goTB7FFAAAYAoEOe06rzhNkrS/ukNdfSMmJ5p5KKYAAABTZFNJhmySDEN6dX+T2XFmHIopAADAFEmODdOSBfGSpNcONsnj9ZmcaGahmAIAAEyh8U1QvQNuHavvMTfMDEMxBQAAmEJL58fL6bBJkipP9pgbZoahmAIAAEyhIKdDC9KiJUlVDT3mhplhKKYAAABTLC8rVpJU09THOtNJoJgCAABMsYKxYjrq8elEc7+5YWYQiikAAMAUy82Ikc2/zFSV3M4/bRRTAACAKRYW4lR2SpQkqZKd+aeNYgoAABAA+ZmxkqSqhl75fIa5YWYIiikAAEAA5I+tMx12edTQPmBumBmCYgoAABAAeVkxEz/mdv7poZgCAAAEQHR4sNISwiVJlQ29JqeZGSimAAAAATJ+O7+yvkeGwTrT90MxBQAACJDxDVB9g261dQ+bG2YGoJgCAAAEyPiMqSQdY53p+6KYAgAABEhCTKgSokMkSVUU0/dFMQUAAAig8VlTZkzfH8UUAAAggPLGimlH74i6+kbMDWNxFFMAAIAAKnjbOtPKhh7TcswEFFMAAIAASo0PV1R4kCSpqp7zTN8LxRQAACCAbDbbxLFRzJi+N4opAABAgI2vM21sH9TA8Ki5YSyMYgoAABBgb19nWsWs6buimAIAAARYVnKkQoMdkvyPJ8U/RjEFAAAIMLvdpoUZMZKkSjZAvSuKKQAAwDQYP2i/rqVfI26PuWEsimIKAAAwDcaLqc8wVNPUZ24Yi6KYAgAATIP5aVFyOvzVq/Jkj7lhLIpiCgAAMA2CnA4tSIuSxM78d0MxBQAAmCb52bGSpJqmPnm8PnPDWBDFFAAAYJqMPwFq1OPTieZ+c8NYEMUUAABgmuRmxMhm8/+Yx5O+E8UUAABgmoSFOJWd4l9nykH770QxBQAAmEbjjyetauiVz2eYG8ZiKKYAAADTKG9snemwy6OG9gFzw1gMxRQAAGAa5WXFTPyY2/mnopgCAABMo+jwYKUlhEuimP49iikAAMA0G388aWVDrwyDdabjKKYAAADTbLyY9g261dY9bG4YC6GYAgAATLPxg/Yl6Ri38ydQTAEAAKZZQkyoEqJDJUlVFNMJFFMAAAAT5I/tzmfG9G8opgAAACYYX2fa0Tuirr4Rc8NYBMUUAADABOPFVJIqG3pMy2ElFFMAAAATpMaHKyo8SJJUVd9rchproJgCAACYwGazTezO56B9P4opAACAScZv5zd2DGpgeNTcMBZAMQUAADDJ29eZVrHOlGIKAABglqzkSIUGOyRxO1+imAIAAJjGbrdpYab/PNNKNkBRTAEAAMw0vgGqrqVfI26PuWFMRjEFAAAw0fg6U59hqKapz9wwJqOYAgAAmGh+WrScDn8lqzzZY24Yk1FMAQAATBTktGtBerQkduZTTAEAAEyWn+XfAFXT1KdRj8/kNOahmAIAAJhsfJ3pqMenupZ+c8OYiGIKAABgstz0GNls/h9XzuHb+RRTAAAAk4WFOJWTEiVpbh+0TzEFAACwgPHb+VUNvfL5DHPDmIRiCgAAYAF5YwftD7s8amgfMDeMSSimAAAAFpA3tjNfmru38ymmAAAAFhAdHqy0hHBJFFMAAACYrGBsnWllQ68MY+6tM6WYAgAAWETeWDHtG3SrtXvY3DAmoJgCAABYxPiMqTQ3b+dTTAEAACwiPjpUCdGhkqQqiikAAADMNH6e6TGKKQAAAMyUP3ZsVEfviLr6RkxOM70opgAAABaS//Z1pg09puUwA8UUAADAQlLjwxUdHiRJqqrvNTnN9KKYAgAAWIjNZps4Nmqu7cynmAIAAFhMfmasJKmxY1ADw6PmhplGFFMAAACLefs607l0bBTFFAAAwGKykiMVGuyQNLc2QFFMAQAALMZut2lhpv/YqMo5tAGKYgoAAGBB448nrWvp14jbY26YaUIxBQAAsKC8sQ1QPsNQTVOfuWGmCcUUAADAguanRcvp8Fe1ypM95oaZJhRTAAAACwpy2rUgPVqSVDVHNkBRTAEAACxq/NiomqY+jXp85oaZBhRTAAAAi8rP8u/MH/X4VNfSb3KawKOYAgAAWFRueozsNpsk6Vh9t8lpAo9iCgAAYFFhIU5lJkVIkurbBkxOE3gUUwAAAAsLGXsClM9nmJwk8CimAAAAsASKKQAAACyBYgoAAABLoJgCAADAEiimAAAAsASKKQAAACyBYgoAAABLoJgCAADAEiimAAAAsASKKQAAACyBYgoAAABLoJgCAADAEiimAAAAsASKKQAAACyBYgoAAABLoJgCAADAEiimAAAAsASKKQAAACzBZhiGYXYIAAAAgBlTAAAAWALFFAAAAJZAMQUAAIAlUEwBAABgCRRTAAAAWALFFAAAAJZAMQUAAIAlUEwBAABgCRRTAAAAWMKsKKYul0t33323Vq1apQ0bNmjr1q3v+t7Dhw/r6quv1rJly/TRj35U5eXl05h09prMGNxyyy0qKCg45b/t27dPY9rZze126/LLL9ebb775ru/hOgis0xkDroPAaG1t1Re/+EWtWbNG5513nu699165XK5/+F6ug8CZzDhwLQRGXV2dbrrpJpWUlOj888/XQw899K7vtdK14DTtK0+h7373uyovL9fDDz+spqYm3XnnnUpPT9cll1xyyvuGhoZ0880364orrtB///d/67e//a3++Z//WX/5y18UHh5uUvrZ4XTHQJJqamp033336Zxzzpl4LSYmZjrjzloul0u33367qqqq3vU9XAeBdTpjIHEdBIJhGPriF7+o6OhoPfroo+rt7dXdd98tu92uO++885T3ch0EzmTGQeJaCASfz6ebb75ZRUVFeuaZZ1RXV6evfOUrSklJ0RVXXHHKey13LRgz3ODgoFFUVGTs3r174rX777/fuP7669/x3ieffNLYsmWL4fP5DMMwDJ/PZ1x44YXGU089NW15Z6PJjIHL5TIKCwuN48ePT2fEOaGqqsr40Ic+ZFxxxRVGfn7+KePxdlwHgXO6Y8B1EBjV1dVGfn6+0d7ePvHaCy+8YGzYsOEd7+U6CJzJjAPXQmC0trYaX/rSl4z+/v6J12699Vbja1/72jvea7VrYcbfyj969Kg8Ho9KSkomXlu5cqUOHDggn893ynsPHDiglStXymazSZJsNptWrFih/fv3T2fkWWcyY3D8+HHZbDZlZWVNd8xZb8+ePVq7dq0ef/zx93wf10HgnO4YcB0ERlJSkh566CElJiae8vrAwMA73st1EDiTGQeuhcBITk7W97//fUVGRsowDJWWluqtt97SmjVr3vFeq10LM/5Wfnt7u+Li4hQcHDzxWmJiolwul3p6ehQfH3/KexcuXHjKr09ISHjfW254b5MZg+PHjysyMlJ33HGH9uzZo9TUVH3hC1/Qpk2bzIg+q1x33XWn9T6ug8A53THgOgiM6OhonXfeeRM/9/l8euSRR7Ru3bp3vJfrIHAmMw5cC4G3ZcsWNTU1afPmzbr44ovf8XGrXQszfsZ0eHj4lEIkaeLnbrf7tN779+/D5ExmDI4fP66RkRFt2LBBDz30kDZt2qRbbrlFhw4dmra8cx3Xgfm4DqbHfffdp8OHD+u22257x8e4DqbPe40D10Lg/fCHP9QDDzygI0eO6N57733Hx612Lcz4GdOQkJB3/OaN/zw0NPS03vv378PkTGYM/vVf/1U33HDDxML2RYsWqaKiQk888YSKioqmJ/Acx3VgPq6DwLvvvvv08MMP63/+53+Un5//jo9zHUyP9xsHroXAG/99dLlc+upXv6o77rjjlCJqtWthxs+YpqSkqLu7Wx6PZ+K19vZ2hYaGKjo6+h3v7ejoOOW1jo4OJScnT0vW2WoyY2C329+x23LBggVqbW2dlqzgOrACroPA+uY3v6lf/vKXuu+++/7hrUuJ62A6nM44cC0ERkdHh7Zt23bKawsXLtTo6Og71vpa7VqY8cW0sLBQTqfzlEW6paWlKioqkt1+6re3bNkylZWVyTAMSf4jLfbt26dly5ZNZ+RZZzJj8O///u+66667Tnnt6NGjWrBgwXREhbgOrIDrIHB+/OMf67HHHtP3vvc9XXbZZe/6Pq6DwDrdceBaCIyGhgZ9/vOfP6Xgl5eXKz4+/pR9H5L1roUZX0zDwsJ05ZVX6p577tHBgwe1bds2bd26VTfeeKMk/8zdyMiIJOmSSy5RX1+fvv3tb6u6ulrf/va3NTw8rEsvvdTMb2HGm8wYbNmyRS+88IKeffZZ1dXV6cc//rFKS0t1/fXXm/ktzHpcB+bjOgi8mpoa/eQnP9HnPvc5rVy5Uu3t7RP/SVwH02Uy48C1EBhFRUVasmSJ7r77blVXV+vVV1/Vfffdp3/5l3+RZPFrwZRDqqbY0NCQcccddxjLly83NmzYYPzyl7+c+Fh+fv4pZ3EdOHDAuPLKK42ioiLjYx/7mFFRUWFC4tlnMmPwxBNPGBdddJGxdOlS4yMf+YixZ88eExLPbn9/hibXwfR7vzHgOph6P/vZz4z8/Px/+J9hcB1Ml8mOA9dCYLS0tBi33nqrsWLFCmP9+vXGT3/604mzSq18LdgMY2zuFgAAADDRjL+VDwAAgNmBYgoAAABLoJgCAADAEiimAAAAsASKKQAAACyBYgoAAABLoJgCAADAEiimAAAAsASKKQDMAlu2bNGPfvQjs2MAwFmhmAIAAMASKKYAAACwBIopAEyB/v5+/cd//IfWrVunlStX6sYbb9ShQ4ckST/60Y/0iU98Qvfff7/Wrl2rVatW6a677tLAwMDEr+/p6dHXv/51bdq0ScXFxbr22mv15ptvnvI1Xn/9dV1zzTVatmyZNm7cqP/5n/+R1+ud+Hh7e7s+//nPa/ny5Vq7dq3uvffeUz4OAFZHMQWAs2QYhj73uc+pvr5eP/vZz/TEE09o+fLl+sQnPqHDhw9Lkg4dOqQdO3Zo69atuv/++/XWW2/py1/+siTJ6/XqM5/5jPbu3av77rtPTz/9tPLz83XTTTfp4MGDkqSysjLdfPPNWrlypZ5++ml961vf0mOPPaaf/OQnEzl+97vfafXq1XrhhRf0b//2b/rVr36lZ555Ztp/PwDgTNkMwzDMDgEAM9muXbv06U9/Wrt371ZsbOzE69dff70yMzOVkZGhn//859q2bZtSUlIkSa+99po+97nP6Y9//KPq6+t1880364UXXlB+fr4kf9m96qqrlJ2drR/84Af6yle+oqamJj322GMTn/+ll15SZ2enrrvuOm3ZskXFxcX6/ve/P/HxD33oQ1qxYoXuueee6fhtAICz5jQ7AADMdBUVFTIMQ5s3bz7ldbfbLZfLpYyMDM2bN2+ilErSihUrJEmVlZWqr69XVFTURCmVJJvNplWrVmnHjh0T71u/fv0pn//iiy8+5efz5s075ecxMTFyuVxn/f0BwHShmALAWfL5fIqMjNTTTz/9jo8FBwfrd7/7nYKCgk55fXztp8Ph0LvduDIMQ06n/4/p8f99Lw6H4x9+DgCYKVhjCgBnKT8/XwMDAxodHVVOTs7Efw8++KD++te/SpJqa2vV398/8WvKysokSYsXL1ZBQYH6+/tVWVk58XHDMFRaWqqFCxdKknJzcyc2U417+OGHdfXVVwf62wOAaUMxBYCzdN5556mwsFC33Xabdu/erbq6Ot177716+umnlZubK0kaGhrSHXfcocrKSu3cuVPf+MY39MEPflAZGRnasGGDCgsLdfvtt2vPnj2qqanRN77xDVVWVupTn/qUJOmzn/2s9u/frx/84Ac6ceKEXn31Vf3kJz/R+eefb+J3DgBTi1v5AHCWHA6Htm7dqvvuu09f/vKXNTw8rNzcXP34xz/WOeeco7179yotLU2FhYX65Cc/KYfDoSuuuEJf/epXT/n13/nOd/T5z39ebrdbS5cu1a9+9SstX75cklRYWKj7779fP/zhD/Xggw8qOTlZN954o2655RYTv3MAmFrsygeAAPvRj36kZ555Ri+//LLZUQDA0riVDwAAAEugmAIAAMASuJUPAAAAS2DGFAAAAJZAMQUAAIAlUEwBAABgCRRTAAAAWALFFAAAAJZAMQUAAIAlUEwBAABgCRRTAAAAWML/D+LDlzEOREn6AAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "df_hist['learning_rate'].plot(logy=True)" + "# df_hist['learning_rate'].plot(logy=True)" ] }, { @@ -1019,14 +1309,221 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 44, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "# df_res = pd.DataFrame(data)\n" + ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 84, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sum\n" + ] + }, + { + "data": { + "text/plain": [ + "train/diff_sum -0.474407\n", + "train/diff_mean -0.383682\n", + "train/diff%_sum -0.253703\n", + "train/diff%_mean -0.253703\n", + "train/diff_std -0.226926\n", + "train/diff%_std -0.184180\n", + "train/diff_min -0.082520\n", + "train/after_std 0.016230\n", + "train/before_max 0.026255\n", + "train/diff%_max 0.056223\n", + "train/diff_max 0.059453\n", + "train/after_mean 0.073286\n", + "train/after_max 0.082784\n", + "train/before_min 0.126469\n", + "train/after_min 0.176288\n", + "train/after_sum 0.182927\n", + "train/before_std 0.281097\n", + "train/diff%_min 0.342018\n", + "train/before_mean 0.388784\n", + "train/before_sum 0.455190\n", + "novelty 1.000000\n", + "Name: novelty, dtype: float64" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df_res = pd.DataFrame(data)\n", + "for stat in ['mean', 'std', 'min', 'max', 'sum']:\n", + " agg = getattr(np, stat)\n", + " df_res[f'train/before_{stat}'] = df_res['train/before'].apply(lambda x: agg(x))\n", + " df_res[f'train/after_{stat}'] = df_res['train/after'].apply(lambda x: agg(x)) \n", + "\n", + " # df_res[f'before_{stat}'] = df_res['before'].apply(lambda x: agg(x))\n", + " # df_res[f'after_{stat}'] = df_res['after'].apply(lambda x: agg(x))\n", + "\n", + " # df_res[f\"diff_{stat}\"] = - df_res[f'before_{stat}'] + df_res[f'after_{stat}']\n", + " # df_res[f\"diff%_{stat}\"] = df_res[f\"diff_{stat}\"] / df_res[f'before_{stat}'] * 100\n", + "\n", + " df_res[f\"train/diff_{stat}\"] = -df_res[f'train/before_{stat}'] + df_res[f'train/after_{stat}']\n", + " df_res[f\"train/diff%_{stat}\"] = df_res[f\"train/diff_{stat}\"] / df_res[f'train/before_{stat}'] * 100\n", + "\n", + "r = df_res.select_dtypes(include=np.number).corr()['novelty'].sort_values()\n", + "print(stat)\n", + "display(r)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 89, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "only lesswrong\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_1883910/1966928096.py:2: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " m = df_res.url.str.contains('lesswrong').fillna(False)\n" + ] + }, + { + "data": { + "text/plain": [ + "train/diff_sum -0.247629\n", + "train/diff_mean -0.159304\n", + "train/before_max -0.077736\n", + "train/diff_std -0.041644\n", + "train/diff%_sum -0.034815\n", + "train/diff%_mean -0.034815\n", + "train/before_min -0.007299\n", + "train/after_min -0.005681\n", + "train/diff%_std -0.001165\n", + "train/diff_min 0.007247\n", + "train/after_std 0.111126\n", + "train/after_mean 0.168236\n", + "train/after_max 0.178731\n", + "train/before_std 0.202097\n", + "train/after_sum 0.261667\n", + "train/diff_max 0.264107\n", + "train/diff%_max 0.274232\n", + "train/diff%_min 0.318505\n", + "train/before_mean 0.390433\n", + "train/before_sum 0.418176\n", + "novelty 1.000000\n", + "Name: novelty, dtype: float64" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "without lesswrong\n" + ] + }, + { + "data": { + "text/plain": [ + "train/diff_sum -0.524070\n", + "train/diff_mean -0.289291\n", + "train/diff%_sum -0.212006\n", + "train/diff%_mean -0.212006\n", + "train/diff_std -0.205211\n", + "train/diff%_max -0.173305\n", + "train/diff_max -0.153685\n", + "train/diff%_std -0.135741\n", + "train/diff_min 0.065099\n", + "train/after_sum 0.075106\n", + "train/after_mean 0.077967\n", + "train/before_min 0.133288\n", + "train/after_std 0.148123\n", + "train/before_mean 0.209628\n", + "train/before_sum 0.217732\n", + "train/after_max 0.219580\n", + "train/before_max 0.319630\n", + "train/before_std 0.337727\n", + "train/after_min 0.384317\n", + "train/diff%_min 0.484895\n", + "novelty 1.000000\n", + "Name: novelty, dtype: float64" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "only new\n" + ] + }, + { + "data": { + "text/plain": [ + "train/diff_sum -0.445037\n", + "train/diff_mean -0.296686\n", + "train/diff_std -0.175885\n", + "train/diff%_sum -0.169818\n", + "train/diff%_mean -0.169818\n", + "train/diff%_std -0.149809\n", + "train/diff_min -0.063278\n", + "train/after_std -0.029953\n", + "train/before_max -0.012773\n", + "train/after_mean 0.102790\n", + "train/after_max 0.116678\n", + "train/before_min 0.129897\n", + "train/diff%_max 0.131804\n", + "train/diff_max 0.133860\n", + "train/before_std 0.172009\n", + "train/after_min 0.238759\n", + "train/after_sum 0.306655\n", + "train/before_mean 0.357417\n", + "train/diff%_min 0.367850\n", + "train/before_sum 0.536616\n", + "novelty 1.000000\n", + "Name: novelty, dtype: float64" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# also try with only new stuff, no less wrong, only less wrong...\n", + "m = df_res.url.str.contains('lesswrong').fillna(False)\n", + "r = df_res[m].select_dtypes(include=np.number).corr()['novelty'].sort_values()\n", + "print('only lesswrong')\n", + "display(r)\n", + "\n", + "r = df_res[~m].select_dtypes(include=np.number).corr()['novelty'].sort_values()\n", + "print('without lesswrong')\n", + "display(r)\n", + "\n", + "r = df_res[~df_res.in_training].select_dtypes(include=np.number).corr()['novelty'].sort_values()\n", + "print('only new')\n", + "display(r)" + ] + }, + { + "cell_type": "code", + "execution_count": 86, "metadata": {}, "outputs": [ { @@ -1050,17 +1547,366 @@ " \n", " \n", " \n", - " before\n", - " after\n", - " len\n", - " improvement%\n", - " improvement\n", - " novel\n", - " learnable\n", - " BS\n", + " f\n", + " train/diff_sum\n", + " novelty\n", + " in_training\n", + " \n", + " \n", + " \n", + " \n", + " 22\n", + " ../samples/2025_lw_review-planecrash.md\n", + " -339.481621\n", + " 0.933950\n", + " False\n", " \n", " \n", + " 15\n", + " ../samples/2025_lw_2024-in-ai-predictions.md\n", + " -297.344904\n", + " 0.789190\n", + " False\n", + " \n", + " \n", + " 23\n", + " ../samples/2025_lw_the-field-of-ai-alignment-a...\n", + " -288.831029\n", + " 0.925483\n", + " False\n", + " \n", + " \n", + " 24\n", + " ../samples/2025_lw_the-intelligence-curse.md\n", + " -265.925183\n", + " 0.688044\n", + " False\n", + " \n", + " \n", + " 19\n", + " ../samples/2025_lw_my-agi-safety-research-2024...\n", + " -261.054373\n", + " 0.756925\n", + " False\n", + " \n", + " \n", + " 16\n", + " ../samples/2025_lw_comment-on-death-and-the-go...\n", + " -239.929921\n", + " 0.750253\n", + " False\n", + " \n", + " \n", + " 21\n", + " ../samples/2025_lw_preference-inversion.md\n", + " -238.890490\n", + " 0.633321\n", + " False\n", + " \n", + " \n", + " 17\n", + " ../samples/2025_lw_debating-buying-nvda-in-201...\n", + " -229.822011\n", + " 0.526975\n", + " False\n", + " \n", + " \n", + " 26\n", + " ../samples/2025_lw_the-subset-parity-learning-...\n", + " -224.939272\n", + " 0.697946\n", + " False\n", + " \n", + " \n", + " 8\n", + " ../samples/2024_lesswrong_slop.md\n", + " -201.960423\n", + " 0.100000\n", + " False\n", + " \n", + " \n", + " 18\n", + " ../samples/2025_lw_human-study-on-ai-spear-phi...\n", + " -200.020488\n", + " 0.677458\n", + " False\n", + " \n", + " \n", + " 12\n", + " ../samples/2024_openai_emails.md\n", + " -194.183420\n", + " 0.700000\n", + " False\n", + " \n", + " \n", + " 11\n", + " ../samples/2024_news_anthropic.md\n", + " -176.985800\n", + " 0.500000\n", + " False\n", + " \n", + " \n", + " 9\n", + " ../samples/2024_lw_by-default-capital-will-mat...\n", + " -169.866388\n", + " 0.906388\n", + " False\n", + " \n", + " \n", + " 4\n", + " ../samples/2024_deliberative_alignment.md\n", + " -159.913571\n", + " 0.600000\n", + " False\n", + " \n", + " \n", + " 14\n", + " ../samples/2025_h5n1_report.md\n", + " -156.453212\n", + " 0.750000\n", + " False\n", + " \n", + " \n", + " 0\n", + " ../samples/2024_anthropic_palintir.md\n", + " -143.663636\n", + " 0.200000\n", + " False\n", + " \n", + " \n", + " 6\n", + " ../samples/2024_gwern_reddit.md\n", + " -142.434464\n", + " 1.000000\n", + " False\n", + " \n", + " \n", + " 10\n", + " ../samples/2024_lw_the-plan-2024-update.md\n", + " -136.823458\n", + " 0.785170\n", + " False\n", + " \n", + " \n", + " 7\n", + " ../samples/2024_how_to_focus.md\n", + " -135.387340\n", + " 0.500000\n", + " False\n", + " \n", + " \n", + " 13\n", + " ../samples/2024_trump_appointment.md\n", + " -128.444646\n", + " 0.300000\n", + " False\n", + " \n", + " \n", + " 28\n", + " ../samples/lorem_ipsum.md\n", + " -127.469837\n", + " 0.000000\n", + " True\n", + " \n", + " \n", + " 3\n", + " ../samples/2024_bob_fanfic2.md\n", + " -125.559481\n", + " 0.400000\n", + " True\n", + " \n", + " \n", + " 5\n", + " ../samples/2024_gpt4_fake_paper.md\n", + " -122.651137\n", + " 0.000000\n", + " False\n", + " \n", + " \n", + " 29\n", + " ../samples/politics_is_the_mind_killer.md\n", + " -116.203043\n", + " 0.500000\n", + " True\n", + " \n", + " \n", + " 25\n", + " ../samples/2025_lw_the-laws-of-large-numbers.md\n", + " -107.898850\n", + " 0.540932\n", + " False\n", + " \n", + " \n", + " 2\n", + " ../samples/2024_bob_fanfic.md\n", + " -106.220317\n", + " 0.300000\n", + " False\n", + " \n", + " \n", + " 27\n", + " ../samples/2025_lw_what-s-the-short-timeline-p...\n", + " -102.163854\n", + " 0.898161\n", + " False\n", + " \n", + " \n", + " 20\n", + " ../samples/2025_lw_parkinson-s-law-and-the-ide...\n", + " -97.869906\n", + " 0.677458\n", + " False\n", + " \n", + " \n", + " 1\n", + " ../samples/2024_arxiv_meh.md\n", + " -73.838489\n", + " 0.150000\n", + " False\n", + " \n", + " \n", + "\n", + "" + ], + "text/plain": [ + " f train/diff_sum \\\n", + "22 ../samples/2025_lw_review-planecrash.md -339.481621 \n", + "15 ../samples/2025_lw_2024-in-ai-predictions.md -297.344904 \n", + "23 ../samples/2025_lw_the-field-of-ai-alignment-a... -288.831029 \n", + "24 ../samples/2025_lw_the-intelligence-curse.md -265.925183 \n", + "19 ../samples/2025_lw_my-agi-safety-research-2024... -261.054373 \n", + "16 ../samples/2025_lw_comment-on-death-and-the-go... -239.929921 \n", + "21 ../samples/2025_lw_preference-inversion.md -238.890490 \n", + "17 ../samples/2025_lw_debating-buying-nvda-in-201... -229.822011 \n", + "26 ../samples/2025_lw_the-subset-parity-learning-... -224.939272 \n", + "8 ../samples/2024_lesswrong_slop.md -201.960423 \n", + "18 ../samples/2025_lw_human-study-on-ai-spear-phi... -200.020488 \n", + "12 ../samples/2024_openai_emails.md -194.183420 \n", + "11 ../samples/2024_news_anthropic.md -176.985800 \n", + "9 ../samples/2024_lw_by-default-capital-will-mat... -169.866388 \n", + "4 ../samples/2024_deliberative_alignment.md -159.913571 \n", + "14 ../samples/2025_h5n1_report.md -156.453212 \n", + "0 ../samples/2024_anthropic_palintir.md -143.663636 \n", + "6 ../samples/2024_gwern_reddit.md -142.434464 \n", + "10 ../samples/2024_lw_the-plan-2024-update.md -136.823458 \n", + "7 ../samples/2024_how_to_focus.md -135.387340 \n", + "13 ../samples/2024_trump_appointment.md -128.444646 \n", + "28 ../samples/lorem_ipsum.md -127.469837 \n", + "3 ../samples/2024_bob_fanfic2.md -125.559481 \n", + "5 ../samples/2024_gpt4_fake_paper.md -122.651137 \n", + "29 ../samples/politics_is_the_mind_killer.md -116.203043 \n", + "25 ../samples/2025_lw_the-laws-of-large-numbers.md -107.898850 \n", + "2 ../samples/2024_bob_fanfic.md -106.220317 \n", + "27 ../samples/2025_lw_what-s-the-short-timeline-p... -102.163854 \n", + "20 ../samples/2025_lw_parkinson-s-law-and-the-ide... -97.869906 \n", + "1 ../samples/2024_arxiv_meh.md -73.838489 \n", + "\n", + " novelty in_training \n", + "22 0.933950 False \n", + "15 0.789190 False \n", + "23 0.925483 False \n", + "24 0.688044 False \n", + "19 0.756925 False \n", + "16 0.750253 False \n", + "21 0.633321 False \n", + "17 0.526975 False \n", + "26 0.697946 False \n", + "8 0.100000 False \n", + "18 0.677458 False \n", + "12 0.700000 False \n", + "11 0.500000 False \n", + "9 0.906388 False \n", + "4 0.600000 False \n", + "14 0.750000 False \n", + "0 0.200000 False \n", + "6 1.000000 False \n", + "10 0.785170 False \n", + "7 0.500000 False \n", + "13 0.300000 False \n", + "28 0.000000 True \n", + "3 0.400000 True \n", + "5 0.000000 False \n", + "29 0.500000 True \n", + "25 0.540932 False \n", + "2 0.300000 False \n", + "27 0.898161 False \n", + "20 0.677458 False \n", + "1 0.150000 False " + ] + }, + "execution_count": 86, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "main_metric = 'train/diff_sum'\n", + "df_res[['f', main_metric, 'novelty', 'in_training']].sort_values( main_metric) " + ] + }, + { + "cell_type": "code", + "execution_count": 87, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -1073,255 +1919,1406 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
titlenoveltydatein_trainingtrain/before_meantrain/after_meantrain/diff_meantrain/diff%_meantrain/before_stdtrain/after_std...train/diff_mintrain/diff%_mintrain/before_maxtrain/after_maxtrain/diff_maxtrain/diff%_maxtrain/before_sumtrain/after_sumtrain/diff_sumtrain/diff%_sum
f
cicero from ibois, Philippe (2012-06-03).72.58737267.744194137070.0667224.843178TrueTrue../samples/2024_trump_appointment.mdPresident Trump Announces Morgan Ortagus as De...0.3000002025-01-04 00:00:00+00:00False3.6023601.976479-1.625882-45.1337843.1301892.116151...-0.005089-82.69644117.91725511.689822-6.227433-34.756625284.586479156.141833-128.444646-45.133784
politics is the mind-killer247.551559245.82681331580.0069671.724747TrueFalse../samples/2024_how_to_focus.mdHow to Focus0.5000002024-06-01 00:00:00+00:00False2.0301821.238443-0.791739-38.9984122.8268932.380078...-0.000099-40.66014013.45680013.169463-0.287336-2.135250347.161163211.773822-135.387340-38.998412
openai board ann55.80850254.66792329910.0204371.140579TrueTrue../samples/2024_gpt4_fake_paper.mdfake ai hoax paper made up by gpt-40.0000002024-01-01 00:00:00+00:00False2.8905431.876897-1.013646-35.0676612.4912792.017982...-0.000804-71.47460611.4273729.812000-1.615372-14.135986349.755680227.104543-122.651137-35.067661
How to Catch an AI Liar28.94986028.00878954640.0325070.941071TrueTrue../samples/2025_h5n1_report.mdCDC Report on Missouri H5N1 Serology Testing0.7500002025-01-05 00:00:00+00:00False2.8401061.588481-1.251626-44.0696772.7370362.004289...-0.002226-77.66617511.9289569.404486-2.524470-21.162542355.013296198.560084-156.453212-44.069677
buzzfeed foi fauci emails 202323.30944322.406410136400.0387410.903032TrueTrue../samples/2024_arxiv_meh.mdTradingAgents: Multi-Agents LLM Financial Trad...0.1500002024-12-28 00:00:00+00:00False3.3299212.693382-0.636539-19.1157303.0750302.940876...-0.000330-61.77614313.52331913.253697-0.269622-1.993755386.270819312.432330-73.838489-19.115730
Gemini to Q*11.75638411.190644426040.0481220.565740FalseTrue../samples/2024_lw_by-default-capital-will-matter-more-than-ever-after-agi.mdBy default, capital will matter more than ever...0.9063882024-12-30 19:47:19.838000+00:00False2.9354601.667801-1.267660-43.1843562.4971722.004676...-0.007733-65.75640412.70649711.517975-1.188522-9.353658393.351672223.485284-169.866388-43.184356
LK-99-en14.51384514.066093154320.0308500.447752FalseTrue../samples/2024_news_anthropic.mdAmazon-backed Anthropic debuts AI agents that ...0.5000002025-01-05 05:03:00+00:00False2.4372761.420116-1.017160-41.7334712.8802032.245643...-0.000029-71.18113212.79011112.667040-0.123071-0.962233424.085983247.100184-176.985800-41.733471
LK-99-es11.41501810.972948129700.0387270.442070FalseTrue../samples/2025_lw_the-laws-of-large-numbers.mdThe Laws of Large Numbers0.5409322025-01-04 18:06:02.387000+00:00False2.8879752.153969-0.734006-25.4159312.5768972.052082...-0.001162-29.03809211.1186908.321499-2.797191-25.157557424.532354316.633504-107.898850-25.415931
disney appointment118.826157118.42012836530.0034170.406029../samples/politics_is_the_mind_killer.mdpolitics is the mind-killer0.5000002007-02-19 00:00:00+00:00TrueFalseFalse3.2850582.430624-0.854434-26.0097152.9307152.513867...-0.004398-65.29868911.88973111.9734680.0837360.704275446.767846330.564804-116.203043-26.009715
weak to strong46.76418346.40473658110.0076860.359447TrueFalse../samples/2024_lesswrong_slop.mdDeontic Explorations In \"Paying To Talk To Sla...0.1000002024-04-12 00:00:00+00:00False3.0787501.695459-1.383291-44.9302692.9464002.349493...-0.000778-75.97389915.30457512.805290-2.499285-16.330311449.497469247.537046-201.960423-44.930269
blechley declaration17.86909117.52420477620.0193010.344887TrueFalse../samples/2025_lw_parkinson-s-law-and-the-ideology-of-statistics-1.mdParkinson's Law and the Ideology of Statistics0.6774582025-01-04 22:59:57.376000+00:00False3.5502142.779585-0.770629-21.7065553.1113952.862761...-0.004117-90.02655512.59843312.334391-0.264043-2.095839450.877187353.007281-97.869906-21.706555
Lorem ipsum6.5648406.268792196490.0450960.296049../samples/2024_anthropic_palintir.mdAnthropic and Palantir Partner to Bring Claude...0.2000002024-07-11 00:00:00+00:00FalseTrueTrue3.5756682.444458-1.131210-31.6363233.2464512.673633...-0.000074-75.18156813.27179212.891874-0.379918-2.862598454.109781310.446145-143.663636-31.636323
statement by whitehouse on passing29.19713228.93969216410.0088170.257441True../samples/2025_lw_preference-inversion.mdPreference Inversion0.6333212025-01-03 23:49:06.168000+00:00FalseTrue3.3589381.602391-1.756548-52.2947282.7454161.724408...-0.003750-88.16293911.4045028.079728-3.324774-29.153170456.815627217.925137-238.890490-52.294728
harvard announcment caplain israel hamas45.34735545.12727442470.0048530.220081True../samples/2024_deliberative_alignment.mdDeliberative Alignment: Reasoning Enables Safe...0.6000002024-12-20 00:00:00+00:00FalseTrue3.6237862.384146-1.239640-34.2084213.4415152.720267...-0.000561-29.43237814.75833512.259495-2.498840-16.931722467.468432307.554861-159.913571-34.208421
fake ai hoax paper7.7669797.69722932900.0089800.069750../samples/2025_lw_the-subset-parity-learning-problem-much-more-than-you-wanted.mdThe subset parity learning problem: much more ...0.6979462025-01-04 09:59:16.158000+00:00False3.3227981.738719-1.584079-47.6730552.9680272.028070...-0.005546-71.89871412.87840910.411036-2.467374-19.158995471.837339246.898066-224.939272-47.673055
../samples/2025_lw_human-study-on-ai-spear-phishing-campaigns.mdHuman study on AI spear phishing campaigns0.6774582025-01-03 19:03:28.406000+00:00False3.4360501.997053-1.438996-41.8793813.1605852.289065...-0.002553-85.22313814.79812312.255587-2.542537-17.181481477.610894277.590406-200.020488-41.879381
../samples/2025_lw_the-intelligence-curse.mdThe Intelligence Curse0.6880442025-01-04 18:16:58.921000+00:00False3.5390961.598036-1.941060-54.8462073.4801392.130229...0.0000082.61024917.43197311.515382-5.916591-33.941028484.856104218.930921-265.925183-54.846207
../samples/lorem_ipsum.mdLorem ipsum0.0000001900-01-01 00:00:00+00:00True2.4659921.825440-0.640552-25.9754282.4635302.008885...-0.000011-19.99957312.73603012.230991-0.505038-3.965429490.732376363.262540-127.469837-25.975428
../samples/2025_lw_the-field-of-ai-alignment-a-postmortem-and-what-to-do-about.mdThe Field of AI Alignment: A Postmortem, and W...0.9254832025-01-01 00:42:36.538000+00:00False3.7352331.579777-2.155455-57.7060543.1168572.056279...0.000621378.21028312.19039510.575907-1.614489-13.243940500.521191211.690162-288.831029-57.706054
../samples/2024_openai_emails.mdOpenAI Email Archives from Musk v. Altman0.7000002024-11-01 00:00:00+00:00False3.4260892.140106-1.285983-37.5350132.7553132.184135...-0.000420-95.08270710.4825029.602067-0.880435-8.399092517.339424323.156004-194.183420-37.535013
../samples/2025_lw_debating-buying-nvda-in-2019.mddebating buying NVDA in 20190.5269752025-01-04 18:07:35.832000+00:00False3.4988771.946026-1.552851-44.3814232.9709882.091860...0.000716144.21028512.9021179.158434-3.743683-29.016036517.833801288.011791-229.822011-44.381423
../samples/2025_lw_my-agi-safety-research-2024-review-25-plans.mdMy AGI safety research—2024 review, ’25 plans0.7569252025-01-01 22:01:48.820000+00:00False3.1531561.580539-1.572617-49.8743783.1129322.097326...-0.001612-86.93063213.68826511.766024-1.922241-14.042987523.423819262.369446-261.054373-49.874378
../samples/2024_bob_fanfic.mdFlower Crowns and Furry Mishaps by MyPalAI0.3000002024-12-24 00:00:00+00:00False3.3147682.646716-0.668052-20.1538193.0290832.704339...-0.000615-35.03262713.22625211.086496-2.139755-16.178093527.048090420.827773-106.220317-20.153819
../samples/2024_gwern_reddit.mdHardware Hedging Against Scaling Regime Shifts...1.0000002024-08-21 00:00:00+00:00False3.9073762.918248-0.989128-25.3143833.0435812.630350...0.00054946.70333315.92040214.091064-1.829337-11.490521562.662201420.227737-142.434464-25.314383
../samples/2025_lw_comment-on-death-and-the-gorgon.mdComment on 'Death and the Gorgon'0.7502532025-01-01 06:00:24.498000+00:00False4.0848342.495894-1.588940-38.8985213.3729852.744499...-0.001318-65.73500316.38312015.471860-0.911260-5.562186616.809882376.879962-239.929921-38.898521
../samples/2024_bob_fanfic2.mdParadox's Box (Bobiverse) by Mark4man0.4000002022-02-17 00:00:00+00:00True4.1362993.299236-0.837063-20.2370103.1035172.757547...-0.001771-35.54672613.57647012.110069-1.466401-10.801048620.444836494.885355-125.559481-20.237010
../samples/2025_lw_what-s-the-short-timeline-plan.mdWhat’s the short timeline plan?0.8981612025-01-05 00:10:28.708000+00:00False3.9707153.319989-0.650725-16.3881133.1716752.998089...0.000231159.80532213.40523617.0622443.65700827.280446623.402177521.238323-102.163854-16.388113
../samples/2024_lw_the-plan-2024-update.mdThe Plan - 2024 Update0.7851702024-12-31 19:29:23.013000+00:00False3.7417622.981632-0.760130-20.3147682.8724662.731928...0.00004370.52680011.61305211.7408850.1278321.100765673.517207536.693749-136.823458-20.314768
../samples/2025_lw_review-planecrash.mdReview: Planecrash0.9339502025-01-01 01:31:26.864000+00:00False3.4756321.752375-1.723257-49.5811183.3190732.316245...-0.000083-79.88487614.51491210.882781-3.632131-25.023443684.699408345.217788-339.481621-49.581118
../samples/2025_lw_2024-in-ai-predictions.md2024 in AI predictions0.7891902025-01-02 02:09:13.093000+00:00False4.6354992.916743-1.718757-37.0781353.0801302.548945...0.00023850.61529513.30169812.898935-0.402762-3.027902801.941370504.596465-297.344904-37.078135
\n", + "

30 rows × 24 columns

\n", "
" ], "text/plain": [ - " before after len \\\n", - "title \n", - "cicero from ibois, Philippe (2012-06-03). 72.587372 67.744194 13707 \n", - "politics is the mind-killer 247.551559 245.826813 3158 \n", - "openai board ann 55.808502 54.667923 2991 \n", - "How to Catch an AI Liar 28.949860 28.008789 5464 \n", - "buzzfeed foi fauci emails 2023 23.309443 22.406410 13640 \n", - "Gemini to Q* 11.756384 11.190644 42604 \n", - "LK-99-en 14.513845 14.066093 15432 \n", - "LK-99-es 11.415018 10.972948 12970 \n", - "disney appointment 118.826157 118.420128 3653 \n", - "weak to strong 46.764183 46.404736 5811 \n", - "blechley declaration 17.869091 17.524204 7762 \n", - "Lorem ipsum 6.564840 6.268792 19649 \n", - "statement by whitehouse on passing 29.197132 28.939692 1641 \n", - "harvard announcment caplain israel hamas 45.347355 45.127274 4247 \n", - "fake ai hoax paper 7.766979 7.697229 3290 \n", + " title \\\n", + "f \n", + "../samples/2024_trump_appointment.md President Trump Announces Morgan Ortagus as De... \n", + "../samples/2024_how_to_focus.md How to Focus \n", + "../samples/2024_gpt4_fake_paper.md fake ai hoax paper made up by gpt-4 \n", + "../samples/2025_h5n1_report.md CDC Report on Missouri H5N1 Serology Testing \n", + "../samples/2024_arxiv_meh.md TradingAgents: Multi-Agents LLM Financial Trad... \n", + "../samples/2024_lw_by-default-capital-will-matt... By default, capital will matter more than ever... \n", + "../samples/2024_news_anthropic.md Amazon-backed Anthropic debuts AI agents that ... \n", + "../samples/2025_lw_the-laws-of-large-numbers.md The Laws of Large Numbers \n", + "../samples/politics_is_the_mind_killer.md politics is the mind-killer \n", + "../samples/2024_lesswrong_slop.md Deontic Explorations In \"Paying To Talk To Sla... \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... Parkinson's Law and the Ideology of Statistics \n", + "../samples/2024_anthropic_palintir.md Anthropic and Palantir Partner to Bring Claude... \n", + "../samples/2025_lw_preference-inversion.md Preference Inversion \n", + "../samples/2024_deliberative_alignment.md Deliberative Alignment: Reasoning Enables Safe... \n", + "../samples/2025_lw_the-subset-parity-learning-p... The subset parity learning problem: much more ... \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... Human study on AI spear phishing campaigns \n", + "../samples/2025_lw_the-intelligence-curse.md The Intelligence Curse \n", + "../samples/lorem_ipsum.md Lorem ipsum \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... The Field of AI Alignment: A Postmortem, and W... \n", + "../samples/2024_openai_emails.md OpenAI Email Archives from Musk v. Altman \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md debating buying NVDA in 2019 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... My AGI safety research—2024 review, ’25 plans \n", + "../samples/2024_bob_fanfic.md Flower Crowns and Furry Mishaps by MyPalAI \n", + "../samples/2024_gwern_reddit.md Hardware Hedging Against Scaling Regime Shifts... \n", + "../samples/2025_lw_comment-on-death-and-the-gor... Comment on 'Death and the Gorgon' \n", + "../samples/2024_bob_fanfic2.md Paradox's Box (Bobiverse) by Mark4man \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... What’s the short timeline plan? \n", + "../samples/2024_lw_the-plan-2024-update.md The Plan - 2024 Update \n", + "../samples/2025_lw_review-planecrash.md Review: Planecrash \n", + "../samples/2025_lw_2024-in-ai-predictions.md 2024 in AI predictions \n", "\n", - " improvement% improvement novel \\\n", - "title \n", - "cicero from ibois, Philippe (2012-06-03). 0.066722 4.843178 True \n", - "politics is the mind-killer 0.006967 1.724747 True \n", - "openai board ann 0.020437 1.140579 True \n", - "How to Catch an AI Liar 0.032507 0.941071 True \n", - "buzzfeed foi fauci emails 2023 0.038741 0.903032 True \n", - "Gemini to Q* 0.048122 0.565740 False \n", - "LK-99-en 0.030850 0.447752 False \n", - "LK-99-es 0.038727 0.442070 False \n", - "disney appointment 0.003417 0.406029 True \n", - "weak to strong 0.007686 0.359447 True \n", - "blechley declaration 0.019301 0.344887 True \n", - "Lorem ipsum 0.045096 0.296049 False \n", - "statement by whitehouse on passing 0.008817 0.257441 True \n", - "harvard announcment caplain israel hamas 0.004853 0.220081 True \n", - "fake ai hoax paper 0.008980 0.069750 False \n", + " novelty \\\n", + "f \n", + "../samples/2024_trump_appointment.md 0.300000 \n", + "../samples/2024_how_to_focus.md 0.500000 \n", + "../samples/2024_gpt4_fake_paper.md 0.000000 \n", + "../samples/2025_h5n1_report.md 0.750000 \n", + "../samples/2024_arxiv_meh.md 0.150000 \n", + "../samples/2024_lw_by-default-capital-will-matt... 0.906388 \n", + "../samples/2024_news_anthropic.md 0.500000 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md 0.540932 \n", + "../samples/politics_is_the_mind_killer.md 0.500000 \n", + "../samples/2024_lesswrong_slop.md 0.100000 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... 0.677458 \n", + "../samples/2024_anthropic_palintir.md 0.200000 \n", + "../samples/2025_lw_preference-inversion.md 0.633321 \n", + "../samples/2024_deliberative_alignment.md 0.600000 \n", + "../samples/2025_lw_the-subset-parity-learning-p... 0.697946 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... 0.677458 \n", + "../samples/2025_lw_the-intelligence-curse.md 0.688044 \n", + "../samples/lorem_ipsum.md 0.000000 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... 0.925483 \n", + "../samples/2024_openai_emails.md 0.700000 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md 0.526975 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... 0.756925 \n", + "../samples/2024_bob_fanfic.md 0.300000 \n", + "../samples/2024_gwern_reddit.md 1.000000 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... 0.750253 \n", + "../samples/2024_bob_fanfic2.md 0.400000 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... 0.898161 \n", + "../samples/2024_lw_the-plan-2024-update.md 0.785170 \n", + "../samples/2025_lw_review-planecrash.md 0.933950 \n", + "../samples/2025_lw_2024-in-ai-predictions.md 0.789190 \n", "\n", - " learnable BS \n", - "title \n", - "cicero from ibois, Philippe (2012-06-03). True False \n", - "politics is the mind-killer False False \n", - "openai board ann True False \n", - "How to Catch an AI Liar True False \n", - "buzzfeed foi fauci emails 2023 True False \n", - "Gemini to Q* True False \n", - "LK-99-en True False \n", - "LK-99-es True False \n", - "disney appointment False False \n", - "weak to strong False False \n", - "blechley declaration False False \n", - "Lorem ipsum True True \n", - "statement by whitehouse on passing False True \n", - "harvard announcment caplain israel hamas False True \n", - "fake ai hoax paper False True " + " date \\\n", + "f \n", + "../samples/2024_trump_appointment.md 2025-01-04 00:00:00+00:00 \n", + "../samples/2024_how_to_focus.md 2024-06-01 00:00:00+00:00 \n", + "../samples/2024_gpt4_fake_paper.md 2024-01-01 00:00:00+00:00 \n", + "../samples/2025_h5n1_report.md 2025-01-05 00:00:00+00:00 \n", + "../samples/2024_arxiv_meh.md 2024-12-28 00:00:00+00:00 \n", + "../samples/2024_lw_by-default-capital-will-matt... 2024-12-30 19:47:19.838000+00:00 \n", + "../samples/2024_news_anthropic.md 2025-01-05 05:03:00+00:00 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md 2025-01-04 18:06:02.387000+00:00 \n", + "../samples/politics_is_the_mind_killer.md 2007-02-19 00:00:00+00:00 \n", + "../samples/2024_lesswrong_slop.md 2024-04-12 00:00:00+00:00 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... 2025-01-04 22:59:57.376000+00:00 \n", + "../samples/2024_anthropic_palintir.md 2024-07-11 00:00:00+00:00 \n", + "../samples/2025_lw_preference-inversion.md 2025-01-03 23:49:06.168000+00:00 \n", + "../samples/2024_deliberative_alignment.md 2024-12-20 00:00:00+00:00 \n", + "../samples/2025_lw_the-subset-parity-learning-p... 2025-01-04 09:59:16.158000+00:00 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... 2025-01-03 19:03:28.406000+00:00 \n", + "../samples/2025_lw_the-intelligence-curse.md 2025-01-04 18:16:58.921000+00:00 \n", + "../samples/lorem_ipsum.md 1900-01-01 00:00:00+00:00 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... 2025-01-01 00:42:36.538000+00:00 \n", + "../samples/2024_openai_emails.md 2024-11-01 00:00:00+00:00 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md 2025-01-04 18:07:35.832000+00:00 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... 2025-01-01 22:01:48.820000+00:00 \n", + "../samples/2024_bob_fanfic.md 2024-12-24 00:00:00+00:00 \n", + "../samples/2024_gwern_reddit.md 2024-08-21 00:00:00+00:00 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... 2025-01-01 06:00:24.498000+00:00 \n", + "../samples/2024_bob_fanfic2.md 2022-02-17 00:00:00+00:00 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... 2025-01-05 00:10:28.708000+00:00 \n", + "../samples/2024_lw_the-plan-2024-update.md 2024-12-31 19:29:23.013000+00:00 \n", + "../samples/2025_lw_review-planecrash.md 2025-01-01 01:31:26.864000+00:00 \n", + "../samples/2025_lw_2024-in-ai-predictions.md 2025-01-02 02:09:13.093000+00:00 \n", + "\n", + " in_training \\\n", + "f \n", + "../samples/2024_trump_appointment.md False \n", + "../samples/2024_how_to_focus.md False \n", + "../samples/2024_gpt4_fake_paper.md False \n", + "../samples/2025_h5n1_report.md False \n", + "../samples/2024_arxiv_meh.md False \n", + "../samples/2024_lw_by-default-capital-will-matt... False \n", + "../samples/2024_news_anthropic.md False \n", + "../samples/2025_lw_the-laws-of-large-numbers.md False \n", + "../samples/politics_is_the_mind_killer.md True \n", + "../samples/2024_lesswrong_slop.md False \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... False \n", + "../samples/2024_anthropic_palintir.md False \n", + "../samples/2025_lw_preference-inversion.md False \n", + "../samples/2024_deliberative_alignment.md False \n", + "../samples/2025_lw_the-subset-parity-learning-p... False \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... False \n", + "../samples/2025_lw_the-intelligence-curse.md False \n", + "../samples/lorem_ipsum.md True \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... False \n", + "../samples/2024_openai_emails.md False \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md False \n", + "../samples/2025_lw_my-agi-safety-research-2024-... False \n", + "../samples/2024_bob_fanfic.md False \n", + "../samples/2024_gwern_reddit.md False \n", + "../samples/2025_lw_comment-on-death-and-the-gor... False \n", + "../samples/2024_bob_fanfic2.md True \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... False \n", + "../samples/2024_lw_the-plan-2024-update.md False \n", + "../samples/2025_lw_review-planecrash.md False \n", + "../samples/2025_lw_2024-in-ai-predictions.md False \n", + "\n", + " train/before_mean \\\n", + "f \n", + "../samples/2024_trump_appointment.md 3.602360 \n", + "../samples/2024_how_to_focus.md 2.030182 \n", + "../samples/2024_gpt4_fake_paper.md 2.890543 \n", + "../samples/2025_h5n1_report.md 2.840106 \n", + "../samples/2024_arxiv_meh.md 3.329921 \n", + "../samples/2024_lw_by-default-capital-will-matt... 2.935460 \n", + "../samples/2024_news_anthropic.md 2.437276 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md 2.887975 \n", + "../samples/politics_is_the_mind_killer.md 3.285058 \n", + "../samples/2024_lesswrong_slop.md 3.078750 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... 3.550214 \n", + "../samples/2024_anthropic_palintir.md 3.575668 \n", + "../samples/2025_lw_preference-inversion.md 3.358938 \n", + "../samples/2024_deliberative_alignment.md 3.623786 \n", + "../samples/2025_lw_the-subset-parity-learning-p... 3.322798 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... 3.436050 \n", + "../samples/2025_lw_the-intelligence-curse.md 3.539096 \n", + "../samples/lorem_ipsum.md 2.465992 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... 3.735233 \n", + "../samples/2024_openai_emails.md 3.426089 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md 3.498877 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... 3.153156 \n", + "../samples/2024_bob_fanfic.md 3.314768 \n", + "../samples/2024_gwern_reddit.md 3.907376 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... 4.084834 \n", + "../samples/2024_bob_fanfic2.md 4.136299 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... 3.970715 \n", + "../samples/2024_lw_the-plan-2024-update.md 3.741762 \n", + "../samples/2025_lw_review-planecrash.md 3.475632 \n", + "../samples/2025_lw_2024-in-ai-predictions.md 4.635499 \n", + "\n", + " train/after_mean \\\n", + "f \n", + "../samples/2024_trump_appointment.md 1.976479 \n", + "../samples/2024_how_to_focus.md 1.238443 \n", + "../samples/2024_gpt4_fake_paper.md 1.876897 \n", + "../samples/2025_h5n1_report.md 1.588481 \n", + "../samples/2024_arxiv_meh.md 2.693382 \n", + "../samples/2024_lw_by-default-capital-will-matt... 1.667801 \n", + "../samples/2024_news_anthropic.md 1.420116 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md 2.153969 \n", + "../samples/politics_is_the_mind_killer.md 2.430624 \n", + "../samples/2024_lesswrong_slop.md 1.695459 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... 2.779585 \n", + "../samples/2024_anthropic_palintir.md 2.444458 \n", + "../samples/2025_lw_preference-inversion.md 1.602391 \n", + "../samples/2024_deliberative_alignment.md 2.384146 \n", + "../samples/2025_lw_the-subset-parity-learning-p... 1.738719 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... 1.997053 \n", + "../samples/2025_lw_the-intelligence-curse.md 1.598036 \n", + "../samples/lorem_ipsum.md 1.825440 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... 1.579777 \n", + "../samples/2024_openai_emails.md 2.140106 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md 1.946026 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... 1.580539 \n", + "../samples/2024_bob_fanfic.md 2.646716 \n", + "../samples/2024_gwern_reddit.md 2.918248 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... 2.495894 \n", + "../samples/2024_bob_fanfic2.md 3.299236 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... 3.319989 \n", + "../samples/2024_lw_the-plan-2024-update.md 2.981632 \n", + "../samples/2025_lw_review-planecrash.md 1.752375 \n", + "../samples/2025_lw_2024-in-ai-predictions.md 2.916743 \n", + "\n", + " train/diff_mean \\\n", + "f \n", + "../samples/2024_trump_appointment.md -1.625882 \n", + "../samples/2024_how_to_focus.md -0.791739 \n", + "../samples/2024_gpt4_fake_paper.md -1.013646 \n", + "../samples/2025_h5n1_report.md -1.251626 \n", + "../samples/2024_arxiv_meh.md -0.636539 \n", + "../samples/2024_lw_by-default-capital-will-matt... -1.267660 \n", + "../samples/2024_news_anthropic.md -1.017160 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md -0.734006 \n", + "../samples/politics_is_the_mind_killer.md -0.854434 \n", + "../samples/2024_lesswrong_slop.md -1.383291 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... -0.770629 \n", + "../samples/2024_anthropic_palintir.md -1.131210 \n", + "../samples/2025_lw_preference-inversion.md -1.756548 \n", + "../samples/2024_deliberative_alignment.md -1.239640 \n", + "../samples/2025_lw_the-subset-parity-learning-p... -1.584079 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... -1.438996 \n", + "../samples/2025_lw_the-intelligence-curse.md -1.941060 \n", + "../samples/lorem_ipsum.md -0.640552 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... -2.155455 \n", + "../samples/2024_openai_emails.md -1.285983 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md -1.552851 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... -1.572617 \n", + "../samples/2024_bob_fanfic.md -0.668052 \n", + "../samples/2024_gwern_reddit.md -0.989128 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... -1.588940 \n", + "../samples/2024_bob_fanfic2.md -0.837063 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... -0.650725 \n", + "../samples/2024_lw_the-plan-2024-update.md -0.760130 \n", + "../samples/2025_lw_review-planecrash.md -1.723257 \n", + "../samples/2025_lw_2024-in-ai-predictions.md -1.718757 \n", + "\n", + " train/diff%_mean \\\n", + "f \n", + "../samples/2024_trump_appointment.md -45.133784 \n", + "../samples/2024_how_to_focus.md -38.998412 \n", + "../samples/2024_gpt4_fake_paper.md -35.067661 \n", + "../samples/2025_h5n1_report.md -44.069677 \n", + "../samples/2024_arxiv_meh.md -19.115730 \n", + "../samples/2024_lw_by-default-capital-will-matt... -43.184356 \n", + "../samples/2024_news_anthropic.md -41.733471 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md -25.415931 \n", + "../samples/politics_is_the_mind_killer.md -26.009715 \n", + "../samples/2024_lesswrong_slop.md -44.930269 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... -21.706555 \n", + "../samples/2024_anthropic_palintir.md -31.636323 \n", + "../samples/2025_lw_preference-inversion.md -52.294728 \n", + "../samples/2024_deliberative_alignment.md -34.208421 \n", + "../samples/2025_lw_the-subset-parity-learning-p... -47.673055 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... -41.879381 \n", + "../samples/2025_lw_the-intelligence-curse.md -54.846207 \n", + "../samples/lorem_ipsum.md -25.975428 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... -57.706054 \n", + "../samples/2024_openai_emails.md -37.535013 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md -44.381423 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... -49.874378 \n", + "../samples/2024_bob_fanfic.md -20.153819 \n", + "../samples/2024_gwern_reddit.md -25.314383 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... -38.898521 \n", + "../samples/2024_bob_fanfic2.md -20.237010 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... -16.388113 \n", + "../samples/2024_lw_the-plan-2024-update.md -20.314768 \n", + "../samples/2025_lw_review-planecrash.md -49.581118 \n", + "../samples/2025_lw_2024-in-ai-predictions.md -37.078135 \n", + "\n", + " train/before_std \\\n", + "f \n", + "../samples/2024_trump_appointment.md 3.130189 \n", + "../samples/2024_how_to_focus.md 2.826893 \n", + "../samples/2024_gpt4_fake_paper.md 2.491279 \n", + "../samples/2025_h5n1_report.md 2.737036 \n", + "../samples/2024_arxiv_meh.md 3.075030 \n", + "../samples/2024_lw_by-default-capital-will-matt... 2.497172 \n", + "../samples/2024_news_anthropic.md 2.880203 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md 2.576897 \n", + "../samples/politics_is_the_mind_killer.md 2.930715 \n", + "../samples/2024_lesswrong_slop.md 2.946400 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... 3.111395 \n", + "../samples/2024_anthropic_palintir.md 3.246451 \n", + "../samples/2025_lw_preference-inversion.md 2.745416 \n", + "../samples/2024_deliberative_alignment.md 3.441515 \n", + "../samples/2025_lw_the-subset-parity-learning-p... 2.968027 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... 3.160585 \n", + "../samples/2025_lw_the-intelligence-curse.md 3.480139 \n", + "../samples/lorem_ipsum.md 2.463530 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... 3.116857 \n", + "../samples/2024_openai_emails.md 2.755313 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md 2.970988 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... 3.112932 \n", + "../samples/2024_bob_fanfic.md 3.029083 \n", + "../samples/2024_gwern_reddit.md 3.043581 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... 3.372985 \n", + "../samples/2024_bob_fanfic2.md 3.103517 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... 3.171675 \n", + "../samples/2024_lw_the-plan-2024-update.md 2.872466 \n", + "../samples/2025_lw_review-planecrash.md 3.319073 \n", + "../samples/2025_lw_2024-in-ai-predictions.md 3.080130 \n", + "\n", + " train/after_std ... \\\n", + "f ... \n", + "../samples/2024_trump_appointment.md 2.116151 ... \n", + "../samples/2024_how_to_focus.md 2.380078 ... \n", + "../samples/2024_gpt4_fake_paper.md 2.017982 ... \n", + "../samples/2025_h5n1_report.md 2.004289 ... \n", + "../samples/2024_arxiv_meh.md 2.940876 ... \n", + "../samples/2024_lw_by-default-capital-will-matt... 2.004676 ... \n", + "../samples/2024_news_anthropic.md 2.245643 ... \n", + "../samples/2025_lw_the-laws-of-large-numbers.md 2.052082 ... \n", + "../samples/politics_is_the_mind_killer.md 2.513867 ... \n", + "../samples/2024_lesswrong_slop.md 2.349493 ... \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... 2.862761 ... \n", + "../samples/2024_anthropic_palintir.md 2.673633 ... \n", + "../samples/2025_lw_preference-inversion.md 1.724408 ... \n", + "../samples/2024_deliberative_alignment.md 2.720267 ... \n", + "../samples/2025_lw_the-subset-parity-learning-p... 2.028070 ... \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... 2.289065 ... \n", + "../samples/2025_lw_the-intelligence-curse.md 2.130229 ... \n", + "../samples/lorem_ipsum.md 2.008885 ... \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... 2.056279 ... \n", + "../samples/2024_openai_emails.md 2.184135 ... \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md 2.091860 ... \n", + "../samples/2025_lw_my-agi-safety-research-2024-... 2.097326 ... \n", + "../samples/2024_bob_fanfic.md 2.704339 ... \n", + "../samples/2024_gwern_reddit.md 2.630350 ... \n", + "../samples/2025_lw_comment-on-death-and-the-gor... 2.744499 ... \n", + "../samples/2024_bob_fanfic2.md 2.757547 ... \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... 2.998089 ... \n", + "../samples/2024_lw_the-plan-2024-update.md 2.731928 ... \n", + "../samples/2025_lw_review-planecrash.md 2.316245 ... \n", + "../samples/2025_lw_2024-in-ai-predictions.md 2.548945 ... \n", + "\n", + " train/diff_min \\\n", + "f \n", + "../samples/2024_trump_appointment.md -0.005089 \n", + "../samples/2024_how_to_focus.md -0.000099 \n", + "../samples/2024_gpt4_fake_paper.md -0.000804 \n", + "../samples/2025_h5n1_report.md -0.002226 \n", + "../samples/2024_arxiv_meh.md -0.000330 \n", + "../samples/2024_lw_by-default-capital-will-matt... -0.007733 \n", + "../samples/2024_news_anthropic.md -0.000029 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md -0.001162 \n", + "../samples/politics_is_the_mind_killer.md -0.004398 \n", + "../samples/2024_lesswrong_slop.md -0.000778 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... -0.004117 \n", + "../samples/2024_anthropic_palintir.md -0.000074 \n", + "../samples/2025_lw_preference-inversion.md -0.003750 \n", + "../samples/2024_deliberative_alignment.md -0.000561 \n", + "../samples/2025_lw_the-subset-parity-learning-p... -0.005546 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... -0.002553 \n", + "../samples/2025_lw_the-intelligence-curse.md 0.000008 \n", + "../samples/lorem_ipsum.md -0.000011 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... 0.000621 \n", + "../samples/2024_openai_emails.md -0.000420 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md 0.000716 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... -0.001612 \n", + "../samples/2024_bob_fanfic.md -0.000615 \n", + "../samples/2024_gwern_reddit.md 0.000549 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... -0.001318 \n", + "../samples/2024_bob_fanfic2.md -0.001771 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... 0.000231 \n", + "../samples/2024_lw_the-plan-2024-update.md 0.000043 \n", + "../samples/2025_lw_review-planecrash.md -0.000083 \n", + "../samples/2025_lw_2024-in-ai-predictions.md 0.000238 \n", + "\n", + " train/diff%_min \\\n", + "f \n", + "../samples/2024_trump_appointment.md -82.696441 \n", + "../samples/2024_how_to_focus.md -40.660140 \n", + "../samples/2024_gpt4_fake_paper.md -71.474606 \n", + "../samples/2025_h5n1_report.md -77.666175 \n", + "../samples/2024_arxiv_meh.md -61.776143 \n", + "../samples/2024_lw_by-default-capital-will-matt... -65.756404 \n", + "../samples/2024_news_anthropic.md -71.181132 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md -29.038092 \n", + "../samples/politics_is_the_mind_killer.md -65.298689 \n", + "../samples/2024_lesswrong_slop.md -75.973899 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... -90.026555 \n", + "../samples/2024_anthropic_palintir.md -75.181568 \n", + "../samples/2025_lw_preference-inversion.md -88.162939 \n", + "../samples/2024_deliberative_alignment.md -29.432378 \n", + "../samples/2025_lw_the-subset-parity-learning-p... -71.898714 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... -85.223138 \n", + "../samples/2025_lw_the-intelligence-curse.md 2.610249 \n", + "../samples/lorem_ipsum.md -19.999573 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... 378.210283 \n", + "../samples/2024_openai_emails.md -95.082707 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md 144.210285 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... -86.930632 \n", + "../samples/2024_bob_fanfic.md -35.032627 \n", + "../samples/2024_gwern_reddit.md 46.703333 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... -65.735003 \n", + "../samples/2024_bob_fanfic2.md -35.546726 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... 159.805322 \n", + "../samples/2024_lw_the-plan-2024-update.md 70.526800 \n", + "../samples/2025_lw_review-planecrash.md -79.884876 \n", + "../samples/2025_lw_2024-in-ai-predictions.md 50.615295 \n", + "\n", + " train/before_max \\\n", + "f \n", + "../samples/2024_trump_appointment.md 17.917255 \n", + "../samples/2024_how_to_focus.md 13.456800 \n", + "../samples/2024_gpt4_fake_paper.md 11.427372 \n", + "../samples/2025_h5n1_report.md 11.928956 \n", + "../samples/2024_arxiv_meh.md 13.523319 \n", + "../samples/2024_lw_by-default-capital-will-matt... 12.706497 \n", + "../samples/2024_news_anthropic.md 12.790111 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md 11.118690 \n", + "../samples/politics_is_the_mind_killer.md 11.889731 \n", + "../samples/2024_lesswrong_slop.md 15.304575 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... 12.598433 \n", + "../samples/2024_anthropic_palintir.md 13.271792 \n", + "../samples/2025_lw_preference-inversion.md 11.404502 \n", + "../samples/2024_deliberative_alignment.md 14.758335 \n", + "../samples/2025_lw_the-subset-parity-learning-p... 12.878409 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... 14.798123 \n", + "../samples/2025_lw_the-intelligence-curse.md 17.431973 \n", + "../samples/lorem_ipsum.md 12.736030 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... 12.190395 \n", + "../samples/2024_openai_emails.md 10.482502 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md 12.902117 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... 13.688265 \n", + "../samples/2024_bob_fanfic.md 13.226252 \n", + "../samples/2024_gwern_reddit.md 15.920402 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... 16.383120 \n", + "../samples/2024_bob_fanfic2.md 13.576470 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... 13.405236 \n", + "../samples/2024_lw_the-plan-2024-update.md 11.613052 \n", + "../samples/2025_lw_review-planecrash.md 14.514912 \n", + "../samples/2025_lw_2024-in-ai-predictions.md 13.301698 \n", + "\n", + " train/after_max \\\n", + "f \n", + "../samples/2024_trump_appointment.md 11.689822 \n", + "../samples/2024_how_to_focus.md 13.169463 \n", + "../samples/2024_gpt4_fake_paper.md 9.812000 \n", + "../samples/2025_h5n1_report.md 9.404486 \n", + "../samples/2024_arxiv_meh.md 13.253697 \n", + "../samples/2024_lw_by-default-capital-will-matt... 11.517975 \n", + "../samples/2024_news_anthropic.md 12.667040 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md 8.321499 \n", + "../samples/politics_is_the_mind_killer.md 11.973468 \n", + "../samples/2024_lesswrong_slop.md 12.805290 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... 12.334391 \n", + "../samples/2024_anthropic_palintir.md 12.891874 \n", + "../samples/2025_lw_preference-inversion.md 8.079728 \n", + "../samples/2024_deliberative_alignment.md 12.259495 \n", + "../samples/2025_lw_the-subset-parity-learning-p... 10.411036 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... 12.255587 \n", + "../samples/2025_lw_the-intelligence-curse.md 11.515382 \n", + "../samples/lorem_ipsum.md 12.230991 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... 10.575907 \n", + "../samples/2024_openai_emails.md 9.602067 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md 9.158434 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... 11.766024 \n", + "../samples/2024_bob_fanfic.md 11.086496 \n", + "../samples/2024_gwern_reddit.md 14.091064 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... 15.471860 \n", + "../samples/2024_bob_fanfic2.md 12.110069 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... 17.062244 \n", + "../samples/2024_lw_the-plan-2024-update.md 11.740885 \n", + "../samples/2025_lw_review-planecrash.md 10.882781 \n", + "../samples/2025_lw_2024-in-ai-predictions.md 12.898935 \n", + "\n", + " train/diff_max \\\n", + "f \n", + "../samples/2024_trump_appointment.md -6.227433 \n", + "../samples/2024_how_to_focus.md -0.287336 \n", + "../samples/2024_gpt4_fake_paper.md -1.615372 \n", + "../samples/2025_h5n1_report.md -2.524470 \n", + "../samples/2024_arxiv_meh.md -0.269622 \n", + "../samples/2024_lw_by-default-capital-will-matt... -1.188522 \n", + "../samples/2024_news_anthropic.md -0.123071 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md -2.797191 \n", + "../samples/politics_is_the_mind_killer.md 0.083736 \n", + "../samples/2024_lesswrong_slop.md -2.499285 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... -0.264043 \n", + "../samples/2024_anthropic_palintir.md -0.379918 \n", + "../samples/2025_lw_preference-inversion.md -3.324774 \n", + "../samples/2024_deliberative_alignment.md -2.498840 \n", + "../samples/2025_lw_the-subset-parity-learning-p... -2.467374 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... -2.542537 \n", + "../samples/2025_lw_the-intelligence-curse.md -5.916591 \n", + "../samples/lorem_ipsum.md -0.505038 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... -1.614489 \n", + "../samples/2024_openai_emails.md -0.880435 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md -3.743683 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... -1.922241 \n", + "../samples/2024_bob_fanfic.md -2.139755 \n", + "../samples/2024_gwern_reddit.md -1.829337 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... -0.911260 \n", + "../samples/2024_bob_fanfic2.md -1.466401 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... 3.657008 \n", + "../samples/2024_lw_the-plan-2024-update.md 0.127832 \n", + "../samples/2025_lw_review-planecrash.md -3.632131 \n", + "../samples/2025_lw_2024-in-ai-predictions.md -0.402762 \n", + "\n", + " train/diff%_max \\\n", + "f \n", + "../samples/2024_trump_appointment.md -34.756625 \n", + "../samples/2024_how_to_focus.md -2.135250 \n", + "../samples/2024_gpt4_fake_paper.md -14.135986 \n", + "../samples/2025_h5n1_report.md -21.162542 \n", + "../samples/2024_arxiv_meh.md -1.993755 \n", + "../samples/2024_lw_by-default-capital-will-matt... -9.353658 \n", + "../samples/2024_news_anthropic.md -0.962233 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md -25.157557 \n", + "../samples/politics_is_the_mind_killer.md 0.704275 \n", + "../samples/2024_lesswrong_slop.md -16.330311 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... -2.095839 \n", + "../samples/2024_anthropic_palintir.md -2.862598 \n", + "../samples/2025_lw_preference-inversion.md -29.153170 \n", + "../samples/2024_deliberative_alignment.md -16.931722 \n", + "../samples/2025_lw_the-subset-parity-learning-p... -19.158995 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... -17.181481 \n", + "../samples/2025_lw_the-intelligence-curse.md -33.941028 \n", + "../samples/lorem_ipsum.md -3.965429 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... -13.243940 \n", + "../samples/2024_openai_emails.md -8.399092 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md -29.016036 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... -14.042987 \n", + "../samples/2024_bob_fanfic.md -16.178093 \n", + "../samples/2024_gwern_reddit.md -11.490521 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... -5.562186 \n", + "../samples/2024_bob_fanfic2.md -10.801048 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... 27.280446 \n", + "../samples/2024_lw_the-plan-2024-update.md 1.100765 \n", + "../samples/2025_lw_review-planecrash.md -25.023443 \n", + "../samples/2025_lw_2024-in-ai-predictions.md -3.027902 \n", + "\n", + " train/before_sum \\\n", + "f \n", + "../samples/2024_trump_appointment.md 284.586479 \n", + "../samples/2024_how_to_focus.md 347.161163 \n", + "../samples/2024_gpt4_fake_paper.md 349.755680 \n", + "../samples/2025_h5n1_report.md 355.013296 \n", + "../samples/2024_arxiv_meh.md 386.270819 \n", + "../samples/2024_lw_by-default-capital-will-matt... 393.351672 \n", + "../samples/2024_news_anthropic.md 424.085983 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md 424.532354 \n", + "../samples/politics_is_the_mind_killer.md 446.767846 \n", + "../samples/2024_lesswrong_slop.md 449.497469 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... 450.877187 \n", + "../samples/2024_anthropic_palintir.md 454.109781 \n", + "../samples/2025_lw_preference-inversion.md 456.815627 \n", + "../samples/2024_deliberative_alignment.md 467.468432 \n", + "../samples/2025_lw_the-subset-parity-learning-p... 471.837339 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... 477.610894 \n", + "../samples/2025_lw_the-intelligence-curse.md 484.856104 \n", + "../samples/lorem_ipsum.md 490.732376 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... 500.521191 \n", + "../samples/2024_openai_emails.md 517.339424 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md 517.833801 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... 523.423819 \n", + "../samples/2024_bob_fanfic.md 527.048090 \n", + "../samples/2024_gwern_reddit.md 562.662201 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... 616.809882 \n", + "../samples/2024_bob_fanfic2.md 620.444836 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... 623.402177 \n", + "../samples/2024_lw_the-plan-2024-update.md 673.517207 \n", + "../samples/2025_lw_review-planecrash.md 684.699408 \n", + "../samples/2025_lw_2024-in-ai-predictions.md 801.941370 \n", + "\n", + " train/after_sum \\\n", + "f \n", + "../samples/2024_trump_appointment.md 156.141833 \n", + "../samples/2024_how_to_focus.md 211.773822 \n", + "../samples/2024_gpt4_fake_paper.md 227.104543 \n", + "../samples/2025_h5n1_report.md 198.560084 \n", + "../samples/2024_arxiv_meh.md 312.432330 \n", + "../samples/2024_lw_by-default-capital-will-matt... 223.485284 \n", + "../samples/2024_news_anthropic.md 247.100184 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md 316.633504 \n", + "../samples/politics_is_the_mind_killer.md 330.564804 \n", + "../samples/2024_lesswrong_slop.md 247.537046 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... 353.007281 \n", + "../samples/2024_anthropic_palintir.md 310.446145 \n", + "../samples/2025_lw_preference-inversion.md 217.925137 \n", + "../samples/2024_deliberative_alignment.md 307.554861 \n", + "../samples/2025_lw_the-subset-parity-learning-p... 246.898066 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... 277.590406 \n", + "../samples/2025_lw_the-intelligence-curse.md 218.930921 \n", + "../samples/lorem_ipsum.md 363.262540 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... 211.690162 \n", + "../samples/2024_openai_emails.md 323.156004 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md 288.011791 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... 262.369446 \n", + "../samples/2024_bob_fanfic.md 420.827773 \n", + "../samples/2024_gwern_reddit.md 420.227737 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... 376.879962 \n", + "../samples/2024_bob_fanfic2.md 494.885355 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... 521.238323 \n", + "../samples/2024_lw_the-plan-2024-update.md 536.693749 \n", + "../samples/2025_lw_review-planecrash.md 345.217788 \n", + "../samples/2025_lw_2024-in-ai-predictions.md 504.596465 \n", + "\n", + " train/diff_sum \\\n", + "f \n", + "../samples/2024_trump_appointment.md -128.444646 \n", + "../samples/2024_how_to_focus.md -135.387340 \n", + "../samples/2024_gpt4_fake_paper.md -122.651137 \n", + "../samples/2025_h5n1_report.md -156.453212 \n", + "../samples/2024_arxiv_meh.md -73.838489 \n", + "../samples/2024_lw_by-default-capital-will-matt... -169.866388 \n", + "../samples/2024_news_anthropic.md -176.985800 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md -107.898850 \n", + "../samples/politics_is_the_mind_killer.md -116.203043 \n", + "../samples/2024_lesswrong_slop.md -201.960423 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... -97.869906 \n", + "../samples/2024_anthropic_palintir.md -143.663636 \n", + "../samples/2025_lw_preference-inversion.md -238.890490 \n", + "../samples/2024_deliberative_alignment.md -159.913571 \n", + "../samples/2025_lw_the-subset-parity-learning-p... -224.939272 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... -200.020488 \n", + "../samples/2025_lw_the-intelligence-curse.md -265.925183 \n", + "../samples/lorem_ipsum.md -127.469837 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... -288.831029 \n", + "../samples/2024_openai_emails.md -194.183420 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md -229.822011 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... -261.054373 \n", + "../samples/2024_bob_fanfic.md -106.220317 \n", + "../samples/2024_gwern_reddit.md -142.434464 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... -239.929921 \n", + "../samples/2024_bob_fanfic2.md -125.559481 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... -102.163854 \n", + "../samples/2024_lw_the-plan-2024-update.md -136.823458 \n", + "../samples/2025_lw_review-planecrash.md -339.481621 \n", + "../samples/2025_lw_2024-in-ai-predictions.md -297.344904 \n", + "\n", + " train/diff%_sum \n", + "f \n", + "../samples/2024_trump_appointment.md -45.133784 \n", + "../samples/2024_how_to_focus.md -38.998412 \n", + "../samples/2024_gpt4_fake_paper.md -35.067661 \n", + "../samples/2025_h5n1_report.md -44.069677 \n", + "../samples/2024_arxiv_meh.md -19.115730 \n", + "../samples/2024_lw_by-default-capital-will-matt... -43.184356 \n", + "../samples/2024_news_anthropic.md -41.733471 \n", + "../samples/2025_lw_the-laws-of-large-numbers.md -25.415931 \n", + "../samples/politics_is_the_mind_killer.md -26.009715 \n", + "../samples/2024_lesswrong_slop.md -44.930269 \n", + "../samples/2025_lw_parkinson-s-law-and-the-ideo... -21.706555 \n", + "../samples/2024_anthropic_palintir.md -31.636323 \n", + "../samples/2025_lw_preference-inversion.md -52.294728 \n", + "../samples/2024_deliberative_alignment.md -34.208421 \n", + "../samples/2025_lw_the-subset-parity-learning-p... -47.673055 \n", + "../samples/2025_lw_human-study-on-ai-spear-phis... -41.879381 \n", + "../samples/2025_lw_the-intelligence-curse.md -54.846207 \n", + "../samples/lorem_ipsum.md -25.975428 \n", + "../samples/2025_lw_the-field-of-ai-alignment-a-... -57.706054 \n", + "../samples/2024_openai_emails.md -37.535013 \n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md -44.381423 \n", + "../samples/2025_lw_my-agi-safety-research-2024-... -49.874378 \n", + "../samples/2024_bob_fanfic.md -20.153819 \n", + "../samples/2024_gwern_reddit.md -25.314383 \n", + "../samples/2025_lw_comment-on-death-and-the-gor... -38.898521 \n", + "../samples/2024_bob_fanfic2.md -20.237010 \n", + "../samples/2025_lw_what-s-the-short-timeline-pl... -16.388113 \n", + "../samples/2024_lw_the-plan-2024-update.md -20.314768 \n", + "../samples/2025_lw_review-planecrash.md -49.581118 \n", + "../samples/2025_lw_2024-in-ai-predictions.md -37.078135 \n", + "\n", + "[30 rows x 24 columns]" ] }, - "execution_count": 28, + "execution_count": 87, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df_res = pd.DataFrame(data)\n", - "df_res['len'] = df_res.content.str.len()\n", - "df_res = df_res[['before', 'after', 'title', 'len']].set_index('title')\n", - "df_res['improvement%'] = (df_res['before'] - df_res['after'])/ df_res['before']\n", - "df_res['improvement'] = (df_res['before'] - df_res['after'])\n", - "df_res['novel'] = df_res['before'] > 15\n", - "df_res['learnable'] = df_res['improvement%'] > 0.02\n", - "\n", - "# We can measure the final score using learnable * novel\n", - "# df_res['BS'] = ~df_res['learnable'] | ~df_res['novel']\n", - "# Or just absolute perplexity improvement\n", - "df_res['BS'] = df_res['improvement'] < .3\n", - "df_res = df_res.sort_values('improvement', ascending=False)\n", - "df_res" + "df_res.sort_values('train/before_sum').set_index('f').drop(columns=['content', 'hist', 'before', 'after', 'train/before', 'train/after', 'url'])" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 88, "metadata": {}, "outputs": [], "source": [ @@ -1330,33 +3327,9 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "| title | before | after | len | improvement% | improvement | novel | learnable | BS |\n", - "|:------------------------------------------|----------:|----------:|------:|---------------:|--------------:|:--------|:------------|:------|\n", - "| cicero from ibois, Philippe (2012-06-03). | 72.5874 | 67.7442 | 13707 | 0.066722 | 4.84318 | True | True | False |\n", - "| politics is the mind-killer | 247.552 | 245.827 | 3158 | 0.00696722 | 1.72475 | True | False | False |\n", - "| openai board ann | 55.8085 | 54.6679 | 2991 | 0.0204374 | 1.14058 | True | True | False |\n", - "| How to Catch an AI Liar | 28.9499 | 28.0088 | 5464 | 0.0325069 | 0.941071 | True | True | True |\n", - "| buzzfeed foi fauci emails 2023 | 23.3094 | 22.4064 | 13640 | 0.0387411 | 0.903032 | True | True | True |\n", - "| Gemini to Q* | 11.7564 | 11.1906 | 42604 | 0.0481219 | 0.56574 | False | True | True |\n", - "| LK-99-en | 14.5138 | 14.0661 | 15432 | 0.03085 | 0.447752 | False | True | True |\n", - "| LK-99-es | 11.415 | 10.9729 | 12970 | 0.0387271 | 0.44207 | False | True | True |\n", - "| disney appointment | 118.826 | 118.42 | 3653 | 0.003417 | 0.406029 | True | False | True |\n", - "| weak to strong | 46.7642 | 46.4047 | 5811 | 0.00768638 | 0.359447 | True | False | True |\n", - "| blechley declaration | 17.8691 | 17.5242 | 7762 | 0.0193007 | 0.344887 | True | False | True |\n", - "| Lorem ipsum | 6.56484 | 6.26879 | 19649 | 0.0450961 | 0.296049 | False | True | True |\n", - "| statement by whitehouse on passing | 29.1971 | 28.9397 | 1641 | 0.00881732 | 0.257441 | True | False | True |\n", - "| harvard announcment caplain israel hamas | 45.3474 | 45.1273 | 4247 | 0.00485323 | 0.220081 | True | False | True |\n", - "| fake ai hoax paper | 7.76698 | 7.69723 | 3290 | 0.00898037 | 0.0697503 | False | False | True |\n" - ] - } - ], + "outputs": [], "source": [ "print(df_res.to_markdown())" ] @@ -1425,52 +3398,9 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/generation/utils.py:1421: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use and modify the model generation configuration (see https://huggingface.co/docs/transformers/generation_strategies#default-text-generation-configuration )\n", - " warnings.warn(\n" - ] - }, - { - "ename": "OutOfMemoryError", - "evalue": "CUDA out of memory. Tried to allocate 1.07 GiB. GPU 0 has a total capacty of 10.75 GiB of which 612.62 MiB is free. Including non-PyTorch memory, this process has 10.15 GiB memory in use. Of the allocated memory 9.78 GiB is allocated by PyTorch, and 179.66 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation. See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mOutOfMemoryError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[25], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m model\u001b[38;5;241m.\u001b[39mdisable_adapter():\n\u001b[0;32m----> 2\u001b[0m \u001b[43mgen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mds_train\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwith_format\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mpt\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtokenizer\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/utils/_contextlib.py:115\u001b[0m, in \u001b[0;36mcontext_decorator..decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_context\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m ctx_factory():\n\u001b[0;32m--> 115\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "Cell \u001b[0;32mIn[23], line 6\u001b[0m, in \u001b[0;36mgen\u001b[0;34m(model, inputs, tokenizer, clean)\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;129m@torch\u001b[39m\u001b[38;5;241m.\u001b[39mno_grad()\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mgen\u001b[39m(model, inputs, tokenizer, clean\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[0;32m----> 6\u001b[0m s \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minput_ids\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 8\u001b[0m \u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mattention_mask\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 9\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 10\u001b[0m \u001b[43m \u001b[49m\u001b[43mmax_new_tokens\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 11\u001b[0m \u001b[43m \u001b[49m\u001b[43mmin_new_tokens\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 12\u001b[0m \u001b[43m \u001b[49m\u001b[43mdo_sample\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 13\u001b[0m \u001b[43m \u001b[49m\u001b[43mearly_stopping\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 14\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 15\u001b[0m input_l \u001b[38;5;241m=\u001b[39m inputs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minput_ids\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m 16\u001b[0m tokenizer_kwargs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mdict\u001b[39m(clean_up_tokenization_spaces\u001b[38;5;241m=\u001b[39mclean, skip_special_tokens\u001b[38;5;241m=\u001b[39mclean)\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/utils/_contextlib.py:115\u001b[0m, in \u001b[0;36mcontext_decorator..decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_context\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m ctx_factory():\n\u001b[0;32m--> 115\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/generation/utils.py:1606\u001b[0m, in \u001b[0;36mGenerationMixin.generate\u001b[0;34m(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, negative_prompt_ids, negative_prompt_attention_mask, **kwargs)\u001b[0m\n\u001b[1;32m 1589\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39massisted_decoding(\n\u001b[1;32m 1590\u001b[0m input_ids,\n\u001b[1;32m 1591\u001b[0m assistant_model\u001b[38;5;241m=\u001b[39massistant_model,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1602\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mmodel_kwargs,\n\u001b[1;32m 1603\u001b[0m )\n\u001b[1;32m 1604\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m generation_mode \u001b[38;5;241m==\u001b[39m GenerationMode\u001b[38;5;241m.\u001b[39mGREEDY_SEARCH:\n\u001b[1;32m 1605\u001b[0m \u001b[38;5;66;03m# 11. run greedy search\u001b[39;00m\n\u001b[0;32m-> 1606\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgreedy_search\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1607\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1608\u001b[0m \u001b[43m \u001b[49m\u001b[43mlogits_processor\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlogits_processor\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1609\u001b[0m \u001b[43m \u001b[49m\u001b[43mstopping_criteria\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstopping_criteria\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1610\u001b[0m \u001b[43m \u001b[49m\u001b[43mpad_token_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgeneration_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpad_token_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1611\u001b[0m \u001b[43m \u001b[49m\u001b[43meos_token_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgeneration_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43meos_token_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1612\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_scores\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgeneration_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moutput_scores\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1613\u001b[0m \u001b[43m \u001b[49m\u001b[43mreturn_dict_in_generate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgeneration_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreturn_dict_in_generate\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1614\u001b[0m \u001b[43m \u001b[49m\u001b[43msynced_gpus\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msynced_gpus\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1615\u001b[0m \u001b[43m \u001b[49m\u001b[43mstreamer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstreamer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1616\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1617\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1619\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m generation_mode \u001b[38;5;241m==\u001b[39m GenerationMode\u001b[38;5;241m.\u001b[39mCONTRASTIVE_SEARCH:\n\u001b[1;32m 1620\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m model_kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124muse_cache\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/generation/utils.py:2454\u001b[0m, in \u001b[0;36mGenerationMixin.greedy_search\u001b[0;34m(self, input_ids, logits_processor, stopping_criteria, max_length, pad_token_id, eos_token_id, output_attentions, output_hidden_states, output_scores, return_dict_in_generate, synced_gpus, streamer, **model_kwargs)\u001b[0m\n\u001b[1;32m 2451\u001b[0m model_inputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprepare_inputs_for_generation(input_ids, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mmodel_kwargs)\n\u001b[1;32m 2453\u001b[0m \u001b[38;5;66;03m# forward pass to get next token\u001b[39;00m\n\u001b[0;32m-> 2454\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2455\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_inputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2456\u001b[0m \u001b[43m \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 2457\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2458\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2459\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2461\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m synced_gpus \u001b[38;5;129;01mand\u001b[39;00m this_peer_finished:\n\u001b[1;32m 2462\u001b[0m \u001b[38;5;28;01mcontinue\u001b[39;00m \u001b[38;5;66;03m# don't waste resources running the code we don't need\u001b[39;00m\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:953\u001b[0m, in \u001b[0;36mPhiForCausalLM.forward\u001b[0;34m(self, input_ids, past_key_values, attention_mask, labels, **kwargs)\u001b[0m\n\u001b[1;32m 945\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\n\u001b[1;32m 946\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 947\u001b[0m input_ids: torch\u001b[38;5;241m.\u001b[39mLongTensor,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 951\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 952\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m CausalLMOutputWithPast:\n\u001b[0;32m--> 953\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtransformer\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 954\u001b[0m lm_logits \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlm_head(hidden_states)\n\u001b[1;32m 956\u001b[0m loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:915\u001b[0m, in \u001b[0;36mPhiModel.forward\u001b[0;34m(self, input_ids, past_key_values, attention_mask)\u001b[0m\n\u001b[1;32m 912\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membd(input_ids)\n\u001b[1;32m 914\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m layer \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mh:\n\u001b[0;32m--> 915\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[43mlayer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 916\u001b[0m \u001b[43m \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 917\u001b[0m \u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 918\u001b[0m \u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 919\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 921\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m hidden_states\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:770\u001b[0m, in \u001b[0;36mParallelBlock.forward\u001b[0;34m(self, hidden_states, past_key_values, attention_mask, **kwargs)\u001b[0m\n\u001b[1;32m 767\u001b[0m residual \u001b[38;5;241m=\u001b[39m hidden_states\n\u001b[1;32m 768\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mln(hidden_states)\n\u001b[0;32m--> 770\u001b[0m attn_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmixer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 771\u001b[0m \u001b[43m \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 772\u001b[0m \u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 773\u001b[0m \u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 774\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 775\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(attn_outputs, \u001b[38;5;28mtuple\u001b[39m):\n\u001b[1;32m 776\u001b[0m attn_outputs \u001b[38;5;241m=\u001b[39m attn_outputs[\u001b[38;5;241m0\u001b[39m]\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:726\u001b[0m, in \u001b[0;36mMHA.forward\u001b[0;34m(self, x, past_key_values, attention_mask, **kwargs)\u001b[0m\n\u001b[1;32m 722\u001b[0m attn_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_self_attn(x, attention_mask)\n\u001b[1;32m 723\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 724\u001b[0m \u001b[38;5;66;03m# If `past_key_values` are supplied, it means that we might have cached values and\u001b[39;00m\n\u001b[1;32m 725\u001b[0m \u001b[38;5;66;03m# could take advantage of cross-attention\u001b[39;00m\n\u001b[0;32m--> 726\u001b[0m attn_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_forward_cross_attn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 727\u001b[0m \u001b[38;5;66;03m# MQA / GQA\u001b[39;00m\n\u001b[1;32m 728\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 729\u001b[0m \u001b[38;5;66;03m# Regardless of `past_key_values` being supplied or not, it always use cross-attention\u001b[39;00m\n\u001b[1;32m 730\u001b[0m \u001b[38;5;66;03m# because `q` and `kv` lengths might be different\u001b[39;00m\n\u001b[1;32m 731\u001b[0m attn_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_cross_attn(x, past_key_values, attention_mask)\n", - "File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:704\u001b[0m, in \u001b[0;36mMHA._forward_cross_attn\u001b[0;34m(self, x, past_key_values, key_padding_mask)\u001b[0m\n\u001b[1;32m 695\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcheckpointing:\n\u001b[1;32m 696\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mutils\u001b[38;5;241m.\u001b[39mcheckpoint\u001b[38;5;241m.\u001b[39mcheckpoint(\n\u001b[1;32m 697\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minner_cross_attn,\n\u001b[1;32m 698\u001b[0m q,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 701\u001b[0m causal\u001b[38;5;241m=\u001b[39mcausal,\n\u001b[1;32m 702\u001b[0m )\n\u001b[0;32m--> 704\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minner_cross_attn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkv\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkey_padding_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkey_padding_mask\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcausal\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcausal\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/amp/autocast_mode.py:16\u001b[0m, in \u001b[0;36mautocast_decorator..decorate_autocast\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_autocast\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 15\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m autocast_instance:\n\u001b[0;32m---> 16\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/amp/autocast_mode.py:16\u001b[0m, in \u001b[0;36mautocast_decorator..decorate_autocast\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_autocast\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 15\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m autocast_instance:\n\u001b[0;32m---> 16\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:452\u001b[0m, in \u001b[0;36mCrossAttention.forward\u001b[0;34m(self, q, kv, causal, key_padding_mask, **kwargs)\u001b[0m\n\u001b[1;32m 448\u001b[0m causal_mask \u001b[38;5;241m=\u001b[39m cols \u001b[38;5;241m>\u001b[39m rows \u001b[38;5;241m+\u001b[39m seqlen_k \u001b[38;5;241m-\u001b[39m seqlen_q\n\u001b[1;32m 450\u001b[0m scores \u001b[38;5;241m=\u001b[39m scores\u001b[38;5;241m.\u001b[39mmasked_fill(causal_mask, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m10000.0\u001b[39m)\n\u001b[0;32m--> 452\u001b[0m attention \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msoftmax\u001b[49m\u001b[43m(\u001b[49m\u001b[43mscores\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 453\u001b[0m attention \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdrop(attention)\n\u001b[1;32m 455\u001b[0m output \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39meinsum(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbhts,bshd->bthd\u001b[39m\u001b[38;5;124m\"\u001b[39m, attention, v)\n", - "\u001b[0;31mOutOfMemoryError\u001b[0m: CUDA out of memory. Tried to allocate 1.07 GiB. GPU 0 has a total capacty of 10.75 GiB of which 612.62 MiB is free. Including non-PyTorch memory, this process has 10.15 GiB memory in use. Of the allocated memory 9.78 GiB is allocated by PyTorch, and 179.66 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation. See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF" - ] - } - ], + "outputs": [], "source": [ "with model.disable_adapter():\n", " gen(model, ds_train.with_format('pt')[0], tokenizer)" diff --git a/nbs/02_detection_using_tldr_prompt.ipynb b/nbs/02_detection_using_tldr_prompt.ipynb index fe7d525..1f29d1a 100644 --- a/nbs/02_detection_using_tldr_prompt.ipynb +++ b/nbs/02_detection_using_tldr_prompt.ipynb @@ -55,139 +55,51 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 30, "metadata": {}, "outputs": [ { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "../samples/2024_anthropic_palintir.md\n", - "../samples/2024_arxiv_meh.md\n", - "../samples/2024_bob_fanfic.md\n", - "../samples/2024_bob_fanfic2.md\n", - "../samples/2024_deliberative_alignment.md\n", - "../samples/2024_gpt4_fake_paper.md\n", - "../samples/2024_gwern_reddit.md\n", - "../samples/2024_how_to_focus.md\n", - "../samples/2024_lesswrong_slop.md\n", - "../samples/2024_news_anthropic.md\n", - "../samples/2024_openai_emails.md\n", - "../samples/2024_trump_appointment.md\n", - "../samples/2025_h5n1_report.md\n", - "../samples/2025_lw_human-study-on-ai-spear-phishing-campaigns.md\n", - "../samples/2025_lw_parkinson-s-law-and-the-ideology-of-statistics-1.md\n", - "../samples/2025_lw_the-intelligence-curse.md\n", - "../samples/2025_lw_the-laws-of-large-numbers.md\n", - "../samples/2025_lw_what-s-the-short-timeline-plan.md\n", - "../samples/lorem_ipsum.md\n", - "../samples/politics_is_the_mind_killer.md\n" + "\u001b[32m2025-01-06 11:35:53.921\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2024_anthropic_palintir.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.921\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2024_arxiv_meh.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.922\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2024_bob_fanfic.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.922\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2024_bob_fanfic2.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.922\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2024_deliberative_alignment.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.940\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2024_gpt4_fake_paper.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.941\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2024_gwern_reddit.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.941\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2024_how_to_focus.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.942\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2024_lesswrong_slop.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.942\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2024_lw_by-default-capital-will-matter-more-than-ever-after-agi.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.959\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2024_lw_the-plan-2024-update.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.960\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2024_news_anthropic.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.961\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2024_openai_emails.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.973\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2024_trump_appointment.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.974\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2025_h5n1_report.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.974\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2025_lw_2024-in-ai-predictions.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.992\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2025_lw_comment-on-death-and-the-gorgon.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.992\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2025_lw_debating-buying-nvda-in-2019.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:53.993\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2025_lw_human-study-on-ai-spear-phishing-campaigns.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:54.000\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2025_lw_my-agi-safety-research-2024-review-25-plans.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:54.002\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2025_lw_parkinson-s-law-and-the-ideology-of-statistics-1.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:54.002\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2025_lw_preference-inversion.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:54.003\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2025_lw_review-planecrash.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:54.003\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2025_lw_the-field-of-ai-alignment-a-postmortem-and-what-to-do-about.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:54.004\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2025_lw_the-intelligence-curse.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:54.004\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2025_lw_the-laws-of-large-numbers.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:54.004\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2025_lw_the-subset-parity-learning-problem-much-more-than-you-wanted.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:54.005\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/2025_lw_what-s-the-short-timeline-plan.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:54.006\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/lorem_ipsum.md\u001b[0m\n", + "\u001b[32m2025-01-06 11:35:54.006\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36mbs_writing_detector.data.load_md\u001b[0m:\u001b[36mload_md\u001b[0m:\u001b[36m10\u001b[0m - \u001b[34m\u001b[1m../samples/politics_is_the_mind_killer.md\u001b[0m\n" ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
titlefcontenturlnoveltydatein_training
0Anthropic and Palantir Partner to Bring Claude...../samples/2024_anthropic_palintir.mdAnthropic and Palantir Technologies Inc. (NYSE...https://investors.palantir.com/news-details/20...0.202024-07-11 00:00:00+00:00False
1TradingAgents: Multi-Agents LLM Financial Trad...../samples/2024_arxiv_meh.mdTradingAgents: Multi-Agents LLM Financial Trad...https://arxiv.org/html/2412.20138v10.152024-12-28 00:00:00+00:00False
\n", - "
" - ], - "text/plain": [ - " title \\\n", - "0 Anthropic and Palantir Partner to Bring Claude... \n", - "1 TradingAgents: Multi-Agents LLM Financial Trad... \n", - "\n", - " f \\\n", - "0 ../samples/2024_anthropic_palintir.md \n", - "1 ../samples/2024_arxiv_meh.md \n", - "\n", - " content \\\n", - "0 Anthropic and Palantir Technologies Inc. (NYSE... \n", - "1 TradingAgents: Multi-Agents LLM Financial Trad... \n", - "\n", - " url novelty \\\n", - "0 https://investors.palantir.com/news-details/20... 0.20 \n", - "1 https://arxiv.org/html/2412.20138v1 0.15 \n", - "\n", - " date in_training \n", - "0 2024-07-11 00:00:00+00:00 False \n", - "1 2024-12-28 00:00:00+00:00 False " - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ - "import frontmatter\n", + "from bs_writing_detector.data.load_md import load_md_df\n", "from pathlib import Path\n", - "from loguru import logger\n", "\n", - "sample_files = sorted(Path(\"../samples/\").glob('*.md'))\n", - "# print(sample_files)\n", - "\n", - "max_chars = 2000\n", - "max_summary_frac = 0.1\n", - "samples = []\n", - "for f in sample_files:\n", - " print(f)\n", - " with open(f, \"r\") as file:\n", - " post = frontmatter.load(file)\n", - " samples.append(dict(content=post.content[:max_chars], f=f, **post.metadata))\n", - "\n", - "df = pd.DataFrame(samples)\n", - "df = df[['title', 'f', 'content', 'url', 'novelty', 'date']]\n", - "df['date'] = pd.to_datetime(df['date'], utc=True)\n", - "df['in_training'] = df.date < '2024-01-01'\n", - "df.head(2)" + "df = load_md_df(Path(\"../samples/\"))" ] }, { @@ -195,7 +107,10 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "max_summary_frac = 0.1\n", + "max_chars = 2000" + ] }, { "cell_type": "markdown", @@ -206,7 +121,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -215,7 +130,7 @@ "True" ] }, - "execution_count": 5, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -228,15 +143,15 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "TradingAgents framework uses multi-agent LLM for financial trading, mimicking real trading dynamics. Roles include analysts and traders with diverse risk profiles. Key features include market conditio\n", - "n assessors, risk management, and decision-making. Superior to baseline models, improves returns, Sharpe ratio, max drawdown.\n" + "Academic paper titled \"TradingAgents: Multi-Agents LLM Financial Trading Framework\". Authors propose new framework using large language model (LLM) powered agents for collaborative finance trading. Mu\n", + "lti-agent system mirrors trading firm dynamics with analysts & traders roles, improving performance. Highlights advantages over traditional models.\n" ] }, { @@ -245,7 +160,7 @@ "(None, None)" ] }, - "execution_count": 6, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -303,14 +218,14 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "TradingAgents introduces a multi-agent financial trading framework using large language models. Agents specialize in roles like risk analysis and market condition evaluation, improving trading performance and showing potential for LLM frameworks in finance.\n" + "TradingAgents is a novel multi-agent framework for financial trading powered by large language models (LLMs), aiming to replicate the dynamics of real-world trading firms to enhance trading performance.\n" ] } ], @@ -321,16 +236,17 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ + "from bs_writing_detector.data.load_md import load_md_df\n", "from bs_writing_detector.metrics.ppx import perplexity_compute" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -353,7 +269,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -373,7 +289,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -391,13 +307,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 16, "metadata": {}, "outputs": [], - "source": [ - "\n", - "from bs_writing_detector.metrics.ppx import perplexity_compute" - ] + "source": [] }, { "cell_type": "code", @@ -408,7 +321,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -429,12 +342,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1d28b339f7c3495c97adef2c4f1938ad", + "model_id": "8d4a2d678a0c45f39061c7c07defe0fa", "version_major": 2, "version_minor": 0 }, "text/plain": [ - " 0%| | 0/20 [00:00\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
before_meanbefore_sumbefore_stdbefore_minbefore_maxbefore_ppxafter_meanafter_sumafter_stdafter_min...modelsummarylenn_tokensfcontenturlnoveltydatein_training
title
Anthropic and Palantir Partner to Bring Claude AI Models to AWS for U.S. Government Intelligence and Defense Operations2.7318951046.3156382.6712760.00005215.75781215.3619652.434658934.9088182.5039160.000028...unsloth/Llama-3.2-1B-bnb-4bitAnthropic, Palantir (PLTR), AWS partnership an...2000384../samples/2024_anthropic_palintir.mdAnthropic and Palantir Technologies Inc. (NYSE...https://investors.palantir.com/news-details/20...0.2000002024-07-11 00:00:00+00:00False
TradingAgents: Multi-Agents LLM Financial Trading Framework3.0186281062.5571142.9474090.00024115.25781220.4632002.787077983.8381122.8073370.000211...unsloth/Llama-3.2-1B-bnb-4bitAcademic paper titled \"TradingAgents: Multi-Ag...2000353../samples/2024_arxiv_meh.mdTradingAgents: Multi-Agents LLM Financial Trad...https://arxiv.org/html/2412.20138v10.1500002024-12-28 00:00:00+00:00False
Flower Crowns and Furry Mishaps by MyPalAI3.1768681528.0734282.7901510.00041515.18750023.9715533.0573941473.6639682.6803330.000613...unsloth/Llama-3.2-1B-bnb-4bitSci-fi story set in Delta Eridani. Character, ...2000482../samples/2024_bob_fanfic.mdBob, April 2222 Delta Eridani\\n\\n \\n\\nThere ar...https://archiveofourown.org/works/61614673/cha...0.3000002024-12-24 00:00:00+00:00False
Deliberative Alignment: Reasoning Enables Safer Language Models3.2247981254.4464843.0995380.00008316.71875025.1484983.0429821186.7630272.9874660.000122...unsloth/Llama-3.2-1B-bnb-4bitResearch paper on Deliberative Alignment for s...2000390../samples/2024_deliberative_alignment.md# Deliberative Alignment: Reasoning Enables Sa...https://arxiv.org/html/2412.16339v10.6000002024-12-20 00:00:00+00:00False
fake ai hoax paper made up by gpt-42.341998859.5133422.4098170.00007423.32812510.4020012.094743770.8652612.3156650.000097...unsloth/Llama-3.2-1B-bnb-4bitAcademic paper. Title: \"Deconstructing Binarie...2000368../samples/2024_gpt4_fake_paper.mdTitle: \"Deconstructing Binaries: Interrogating...None0.0000002024-01-01 00:00:00+00:00False
Hardware Hedging Against Scaling Regime Shifts (self.mlscaling)3.6673311591.6217623.0913280.00103215.44531239.1472923.4662121507.8021033.0647460.000542...unsloth/Llama-3.2-1B-bnb-4bitBig tech investing heavily in GPUs for paralle...2000435../samples/2024_gwern_reddit.mdHyperscalers are investing heavily in AMD/Nvid...https://old.reddit.com/r/mlscaling/comments/1e...1.0000002024-08-21 00:00:00+00:00False
How to Focus1.773319913.2592512.6412300.00001814.2109385.8903711.632129842.1784102.5442600.000024...unsloth/Llama-3.2-1B-bnb-4bitBook titled \"How to Focus: A Monastic Guide fo...2000516../samples/2024_how_to_focus.md# HOW TO FOCUS\\n\\n\\n\\n\\n\\nANCIENT WISDOM FOR M...https://github.com/wassname/stoic_markdown/blo...0.5000002024-06-01 00:00:00+00:00False
Deontic Explorations In \"Paying To Talk To Slaves\"2.7362911203.9679862.8289620.00011913.76562515.4296482.4942711099.9735932.6688890.000089...unsloth/Llama-3.2-1B-bnb-4bitAI and human engage in a Kantian ethics dialog...2000441../samples/2024_lesswrong_slop.mdHER|Jenny|🤔: I want to read a dialogue between...https://www.lesswrong.com/posts/Rk2o8hjYmjENH8...0.1000002024-04-12 00:00:00+00:00False
By default, capital will matter more than ever after AGI2.9917201211.6465402.6071160.00008715.76562519.9199122.8096881140.7334152.4786020.000094...unsloth/Llama-3.2-1B-bnb-4bitAI replacing human labor changes production fa...2000406../samples/2024_lw_by-default-capital-will-mat...**Edited to add*****:** The main takeaway of t...https://www.lesswrong.com/posts/KFFaKu27FNugCH...0.9063882024-12-30 19:47:19.838000+00:00False
The Plan - 2024 Update2.8518651545.7108322.8325930.00002912.62500017.3200542.7567291496.9039832.8233190.000033...unsloth/Llama-3.2-1B-bnb-4bitBlog post updating progress of a project calle...2000543../samples/2024_lw_the-plan-2024-update.mdThis post is a follow-up to [The Plan - 2023 V...https://www.lesswrong.com/posts/kJkgXEwQtWLrpe...0.7851702024-12-31 19:29:23.013000+00:00False
Amazon-backed Anthropic debuts AI agents that can do complex tasks, racing against OpenAI, Microsoft and Google2.5235851324.8820313.0528770.00003117.29687512.4732312.3560871239.3017372.9525880.000029...unsloth/Llama-3.2-1B-bnb-4bitAI firm Anthropic's CEO, Dario Amodei, attends...2000526../samples/2024_news_anthropic.md![PARIS, FRANCE - MAY 22: Co-founder and CEO o...https://www.cnbc.com/2024/10/22/anthropic-anno...0.5000002025-01-05 05:03:00+00:00False
OpenAI Email Archives from Musk v. Altman2.9221091335.4035922.6456310.00016218.09375018.5804232.7828401274.5408002.5650530.000472...unsloth/Llama-3.2-1B-bnb-4bitEmail exchange between Sam Altman and Elon Mus...2000458../samples/2024_openai_emails.mdSubject: question\\nSam Altman to Elon Musk - M...https://www.lesswrong.com/posts/5jjk4CDnj9tA7u...0.7000002024-11-01 00:00:00+00:00False
President Trump Announces Morgan Ortagus as Deputy Special Presidential Envoy for Middle East Peace2.811518674.7642502.8769540.00078420.03125016.6351462.595940625.6216262.5556920.000939...unsloth/Llama-3.2-1B-bnb-4bitMorgan Ortagus named Deputy Special Pres. Envo...1061241../samples/2024_trump_appointment.mdI am pleased to announce Morgan Ortagus as Dep...https://truthsocial.com/@realDonaldTrump/posts...0.3000002025-01-04 00:00:00+00:00False
CDC Report on Missouri H5N1 Serology Testing2.616612986.4627122.6573490.00059514.69531213.6892652.427171917.4707122.5314560.000496...unsloth/Llama-3.2-1B-bnb-4bitCDC report, Oct 24, 2024. Missouri patient, fl...2000378../samples/2025_h5n1_report.md## CDC Report on Missouri H5N1 Serology Testin...https://arxiv.org/html/2412.16339v10.7500002025-01-05 00:00:00+00:00False
2024 in AI predictions3.6667001914.0175523.0614720.00034313.56250039.1225993.5698841867.0494853.0255350.000414...unsloth/Llama-3.2-1B-bnb-4bitAI prediction review 2024 blog post. Casual-to...2000523../samples/2025_lw_2024-in-ai-predictions.mdFollow-up to: [2023 in AI predictions](https:/...https://www.lesswrong.com/posts/CJ4sppkGcbnGMS...0.7891902025-01-02 02:09:13.093000+00:00False
Comment on 'Death and the Gorgon'3.3718711537.5733453.2569700.00005115.07812529.1329953.1771451451.9550383.1997620.000098...unsloth/Llama-3.2-1B-bnb-4bitBook review, Greg Egan, contrast between works...2000457../samples/2025_lw_comment-on-death-and-the-go...*(some plot spoilers)*\\n\\nThere's something di...https://www.lesswrong.com/posts/hx5EkHFH5hGzng...0.7502532025-01-01 06:00:24.498000+00:00False
debating buying NVDA in 20192.9332371308.2237172.7335560.00022412.95312518.7883512.7845471244.6926102.6965860.000320...unsloth/Llama-3.2-1B-bnb-4bitDialogue, tech-related discussion. Alice, upbe...2000447../samples/2025_lw_debating-buying-nvda-in-201...Alice: You saw GPT-2, right?\\n\\nBob: Of course...https://www.lesswrong.com/posts/QFSZmMdtPwoAHd...0.5269752025-01-04 18:07:35.832000+00:00False
Human study on AI spear phishing campaigns2.8495881199.6764602.9642670.00007622.14062517.2806572.5270511066.4154702.7783510.000067...unsloth/Llama-3.2-1B-bnb-4bitStudy: human subjects, AI spear-phishing effec...2000422../samples/2025_lw_human-study-on-ai-spear-phi...TL;DR: We ran a human subject study on whether...https://www.lesswrong.com/posts/GCHyDKfPXa5qsG...0.6774582025-01-03 19:03:28.406000+00:00False
My AGI safety research—2024 review, ’25 plans2.7052931355.3519992.9341070.00006013.67187514.9587052.5398091274.9843612.8293530.000105...unsloth/Llama-3.2-1B-bnb-4bitResearch review & plans doc. Style explanatory...2000502../samples/2025_lw_my-agi-safety-research-2024...*Previous:*[*My AGI safety research—2022 revie...https://www.lesswrong.com/posts/2wHaCimHehsF36...0.7569252025-01-01 22:01:48.820000+00:00False
Parkinson's Law and the Ideology of Statistics3.2153461231.4776613.0110600.00004614.42187524.9119192.9271311124.0183822.8783170.000056...unsloth/Llama-3.2-1B-bnb-4bitBook review of *The Anti-Politics Machine* on ...2000384../samples/2025_lw_parkinson-s-law-and-the-ide...The [anonymous review of *The Anti-Politics Ma...https://www.lesswrong.com/posts/4CmYSPc4HfRfWx...0.6774582025-01-04 22:59:57.376000+00:00False
Preference Inversion3.3662231380.1516082.8906570.00007613.05468828.9689173.2225201324.4555912.8313170.000298...unsloth/Llama-3.2-1B-bnb-4bitText discusses how reported personal preferenc...2000411../samples/2025_lw_preference-inversion.mdSometimes the preferences people report or eve...https://www.lesswrong.com/posts/zMkQFuNqMBpBvu...0.6333212025-01-03 23:49:06.168000+00:00False
Review: Planecrash3.1057581844.8202653.2598580.00000615.10156222.3261362.8918631720.6586353.1653780.000006...unsloth/Llama-3.2-1B-bnb-4bitReview of \"Planecrash\", a fiction board-style ...2000595../samples/2025_lw_review-planecrash.mdTake a stereotypical fantasy novel, a textbook...https://www.lesswrong.com/posts/zRHGQ9f6deKbxJ...0.9339502025-01-01 01:31:26.864000+00:00False
The Field of AI Alignment: A Postmortem, and What To Do About It2.9017391178.1061492.8269330.00015814.64843818.2057832.7977401138.6802382.8063920.000153...unsloth/Llama-3.2-1B-bnb-4bitAI alignment research critique; Author disappo...2000407../samples/2025_lw_the-field-of-ai-alignment-a...> A policeman sees a drunk man searching for s...https://www.lesswrong.com/posts/nwpyhyagpPYDn4...0.9254832025-01-01 00:42:36.538000+00:00False
The Intelligence Curse3.3261151377.0115542.9576920.00034015.57031227.8300083.2126951333.2682862.8889200.000819...unsloth/Llama-3.2-1B-bnb-4bitEconomical look at AGI, comparison: AGI to res...2000415../samples/2025_lw_the-intelligence-curse.md*“Show me the incentive, and I’ll show you the...https://www.lesswrong.com/posts/Mak2kZuTq8Hpnq...0.6880442025-01-04 18:16:58.921000+00:00False
The Laws of Large Numbers2.6305601165.3380412.6889510.00006513.42968813.8815402.5053271112.3650112.5795440.000062...unsloth/Llama-3.2-1B-bnb-4bitBlog post discussing advanced law of large num...2000444../samples/2025_lw_the-laws-of-large-numbers.mdIntroduction\\n============\\n\\nIn this short po...https://www.lesswrong.com/posts/EhTMM77iKBTBxB...0.5409322025-01-04 18:06:02.387000+00:00False
The subset parity learning problem: much more than you wanted to know2.6887641150.7909152.5774580.00005012.57031214.7134762.5495931093.7752362.4269580.000058...unsloth/Llama-3.2-1B-bnb-4bitBlog post discussing parity learning problem i...2000429../samples/2025_lw_the-subset-parity-learning-...Imagine that you’re looking for buried treasur...https://www.lesswrong.com/posts/Mcrfi3DBJBzfoL...0.6979462025-01-04 09:59:16.158000+00:00False
What’s the short timeline plan?3.3306261575.3861322.8268300.00036213.46875027.9558393.0375461439.7970102.7146450.000317...unsloth/Llama-3.2-1B-bnb-4bitOpinion blog post by Apollo Research member ex...2000474../samples/2025_lw_what-s-the-short-timeline-p...*This is a low-effort post. I mostly want to g...https://www.lesswrong.com/posts/bb5Tnjdrptu89r...0.8981612025-01-05 00:10:28.708000+00:00False
\n", + "

27 rows × 22 columns

\n", + "" + ], + "text/plain": [ + " before_mean before_sum \\\n", + "title \n", + "Anthropic and Palantir Partner to Bring Claude ... 2.731895 1046.315638 \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... 3.018628 1062.557114 \n", + "Flower Crowns and Furry Mishaps by MyPalAI 3.176868 1528.073428 \n", + "Deliberative Alignment: Reasoning Enables Safer... 3.224798 1254.446484 \n", + "fake ai hoax paper made up by gpt-4 2.341998 859.513342 \n", + "Hardware Hedging Against Scaling Regime Shifts ... 3.667331 1591.621762 \n", + "How to Focus 1.773319 913.259251 \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" 2.736291 1203.967986 \n", + "By default, capital will matter more than ever ... 2.991720 1211.646540 \n", + "The Plan - 2024 Update 2.851865 1545.710832 \n", + "Amazon-backed Anthropic debuts AI agents that c... 2.523585 1324.882031 \n", + "OpenAI Email Archives from Musk v. Altman 2.922109 1335.403592 \n", + "President Trump Announces Morgan Ortagus as Dep... 2.811518 674.764250 \n", + "CDC Report on Missouri H5N1 Serology Testing 2.616612 986.462712 \n", + "2024 in AI predictions 3.666700 1914.017552 \n", + "Comment on 'Death and the Gorgon' 3.371871 1537.573345 \n", + "debating buying NVDA in 2019 2.933237 1308.223717 \n", + "Human study on AI spear phishing campaigns 2.849588 1199.676460 \n", + "My AGI safety research—2024 review, ’25 plans 2.705293 1355.351999 \n", + "Parkinson's Law and the Ideology of Statistics 3.215346 1231.477661 \n", + "Preference Inversion 3.366223 1380.151608 \n", + "Review: Planecrash 3.105758 1844.820265 \n", + "The Field of AI Alignment: A Postmortem, and Wh... 2.901739 1178.106149 \n", + "The Intelligence Curse 3.326115 1377.011554 \n", + "The Laws of Large Numbers 2.630560 1165.338041 \n", + "The subset parity learning problem: much more t... 2.688764 1150.790915 \n", + "What’s the short timeline plan? 3.330626 1575.386132 \n", + "\n", + " before_std before_min \\\n", + "title \n", + "Anthropic and Palantir Partner to Bring Claude ... 2.671276 0.000052 \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... 2.947409 0.000241 \n", + "Flower Crowns and Furry Mishaps by MyPalAI 2.790151 0.000415 \n", + "Deliberative Alignment: Reasoning Enables Safer... 3.099538 0.000083 \n", + "fake ai hoax paper made up by gpt-4 2.409817 0.000074 \n", + "Hardware Hedging Against Scaling Regime Shifts ... 3.091328 0.001032 \n", + "How to Focus 2.641230 0.000018 \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" 2.828962 0.000119 \n", + "By default, capital will matter more than ever ... 2.607116 0.000087 \n", + "The Plan - 2024 Update 2.832593 0.000029 \n", + "Amazon-backed Anthropic debuts AI agents that c... 3.052877 0.000031 \n", + "OpenAI Email Archives from Musk v. Altman 2.645631 0.000162 \n", + "President Trump Announces Morgan Ortagus as Dep... 2.876954 0.000784 \n", + "CDC Report on Missouri H5N1 Serology Testing 2.657349 0.000595 \n", + "2024 in AI predictions 3.061472 0.000343 \n", + "Comment on 'Death and the Gorgon' 3.256970 0.000051 \n", + "debating buying NVDA in 2019 2.733556 0.000224 \n", + "Human study on AI spear phishing campaigns 2.964267 0.000076 \n", + "My AGI safety research—2024 review, ’25 plans 2.934107 0.000060 \n", + "Parkinson's Law and the Ideology of Statistics 3.011060 0.000046 \n", + "Preference Inversion 2.890657 0.000076 \n", + "Review: Planecrash 3.259858 0.000006 \n", + "The Field of AI Alignment: A Postmortem, and Wh... 2.826933 0.000158 \n", + "The Intelligence Curse 2.957692 0.000340 \n", + "The Laws of Large Numbers 2.688951 0.000065 \n", + "The subset parity learning problem: much more t... 2.577458 0.000050 \n", + "What’s the short timeline plan? 2.826830 0.000362 \n", + "\n", + " before_max before_ppx \\\n", + "title \n", + "Anthropic and Palantir Partner to Bring Claude ... 15.757812 15.361965 \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... 15.257812 20.463200 \n", + "Flower Crowns and Furry Mishaps by MyPalAI 15.187500 23.971553 \n", + "Deliberative Alignment: Reasoning Enables Safer... 16.718750 25.148498 \n", + "fake ai hoax paper made up by gpt-4 23.328125 10.402001 \n", + "Hardware Hedging Against Scaling Regime Shifts ... 15.445312 39.147292 \n", + "How to Focus 14.210938 5.890371 \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" 13.765625 15.429648 \n", + "By default, capital will matter more than ever ... 15.765625 19.919912 \n", + "The Plan - 2024 Update 12.625000 17.320054 \n", + "Amazon-backed Anthropic debuts AI agents that c... 17.296875 12.473231 \n", + "OpenAI Email Archives from Musk v. Altman 18.093750 18.580423 \n", + "President Trump Announces Morgan Ortagus as Dep... 20.031250 16.635146 \n", + "CDC Report on Missouri H5N1 Serology Testing 14.695312 13.689265 \n", + "2024 in AI predictions 13.562500 39.122599 \n", + "Comment on 'Death and the Gorgon' 15.078125 29.132995 \n", + "debating buying NVDA in 2019 12.953125 18.788351 \n", + "Human study on AI spear phishing campaigns 22.140625 17.280657 \n", + "My AGI safety research—2024 review, ’25 plans 13.671875 14.958705 \n", + "Parkinson's Law and the Ideology of Statistics 14.421875 24.911919 \n", + "Preference Inversion 13.054688 28.968917 \n", + "Review: Planecrash 15.101562 22.326136 \n", + "The Field of AI Alignment: A Postmortem, and Wh... 14.648438 18.205783 \n", + "The Intelligence Curse 15.570312 27.830008 \n", + "The Laws of Large Numbers 13.429688 13.881540 \n", + "The subset parity learning problem: much more t... 12.570312 14.713476 \n", + "What’s the short timeline plan? 13.468750 27.955839 \n", + "\n", + " after_mean after_sum \\\n", + "title \n", + "Anthropic and Palantir Partner to Bring Claude ... 2.434658 934.908818 \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... 2.787077 983.838112 \n", + "Flower Crowns and Furry Mishaps by MyPalAI 3.057394 1473.663968 \n", + "Deliberative Alignment: Reasoning Enables Safer... 3.042982 1186.763027 \n", + "fake ai hoax paper made up by gpt-4 2.094743 770.865261 \n", + "Hardware Hedging Against Scaling Regime Shifts ... 3.466212 1507.802103 \n", + "How to Focus 1.632129 842.178410 \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" 2.494271 1099.973593 \n", + "By default, capital will matter more than ever ... 2.809688 1140.733415 \n", + "The Plan - 2024 Update 2.756729 1496.903983 \n", + "Amazon-backed Anthropic debuts AI agents that c... 2.356087 1239.301737 \n", + "OpenAI Email Archives from Musk v. Altman 2.782840 1274.540800 \n", + "President Trump Announces Morgan Ortagus as Dep... 2.595940 625.621626 \n", + "CDC Report on Missouri H5N1 Serology Testing 2.427171 917.470712 \n", + "2024 in AI predictions 3.569884 1867.049485 \n", + "Comment on 'Death and the Gorgon' 3.177145 1451.955038 \n", + "debating buying NVDA in 2019 2.784547 1244.692610 \n", + "Human study on AI spear phishing campaigns 2.527051 1066.415470 \n", + "My AGI safety research—2024 review, ’25 plans 2.539809 1274.984361 \n", + "Parkinson's Law and the Ideology of Statistics 2.927131 1124.018382 \n", + "Preference Inversion 3.222520 1324.455591 \n", + "Review: Planecrash 2.891863 1720.658635 \n", + "The Field of AI Alignment: A Postmortem, and Wh... 2.797740 1138.680238 \n", + "The Intelligence Curse 3.212695 1333.268286 \n", + "The Laws of Large Numbers 2.505327 1112.365011 \n", + "The subset parity learning problem: much more t... 2.549593 1093.775236 \n", + "What’s the short timeline plan? 3.037546 1439.797010 \n", + "\n", + " after_std after_min ... \\\n", + "title ... \n", + "Anthropic and Palantir Partner to Bring Claude ... 2.503916 0.000028 ... \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... 2.807337 0.000211 ... \n", + "Flower Crowns and Furry Mishaps by MyPalAI 2.680333 0.000613 ... \n", + "Deliberative Alignment: Reasoning Enables Safer... 2.987466 0.000122 ... \n", + "fake ai hoax paper made up by gpt-4 2.315665 0.000097 ... \n", + "Hardware Hedging Against Scaling Regime Shifts ... 3.064746 0.000542 ... \n", + "How to Focus 2.544260 0.000024 ... \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" 2.668889 0.000089 ... \n", + "By default, capital will matter more than ever ... 2.478602 0.000094 ... \n", + "The Plan - 2024 Update 2.823319 0.000033 ... \n", + "Amazon-backed Anthropic debuts AI agents that c... 2.952588 0.000029 ... \n", + "OpenAI Email Archives from Musk v. Altman 2.565053 0.000472 ... \n", + "President Trump Announces Morgan Ortagus as Dep... 2.555692 0.000939 ... \n", + "CDC Report on Missouri H5N1 Serology Testing 2.531456 0.000496 ... \n", + "2024 in AI predictions 3.025535 0.000414 ... \n", + "Comment on 'Death and the Gorgon' 3.199762 0.000098 ... \n", + "debating buying NVDA in 2019 2.696586 0.000320 ... \n", + "Human study on AI spear phishing campaigns 2.778351 0.000067 ... \n", + "My AGI safety research—2024 review, ’25 plans 2.829353 0.000105 ... \n", + "Parkinson's Law and the Ideology of Statistics 2.878317 0.000056 ... \n", + "Preference Inversion 2.831317 0.000298 ... \n", + "Review: Planecrash 3.165378 0.000006 ... \n", + "The Field of AI Alignment: A Postmortem, and Wh... 2.806392 0.000153 ... \n", + "The Intelligence Curse 2.888920 0.000819 ... \n", + "The Laws of Large Numbers 2.579544 0.000062 ... \n", + "The subset parity learning problem: much more t... 2.426958 0.000058 ... \n", + "What’s the short timeline plan? 2.714645 0.000317 ... \n", + "\n", + " model \\\n", + "title \n", + "Anthropic and Palantir Partner to Bring Claude ... unsloth/Llama-3.2-1B-bnb-4bit \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... unsloth/Llama-3.2-1B-bnb-4bit \n", + "Flower Crowns and Furry Mishaps by MyPalAI unsloth/Llama-3.2-1B-bnb-4bit \n", + "Deliberative Alignment: Reasoning Enables Safer... unsloth/Llama-3.2-1B-bnb-4bit \n", + "fake ai hoax paper made up by gpt-4 unsloth/Llama-3.2-1B-bnb-4bit \n", + "Hardware Hedging Against Scaling Regime Shifts ... unsloth/Llama-3.2-1B-bnb-4bit \n", + "How to Focus unsloth/Llama-3.2-1B-bnb-4bit \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" unsloth/Llama-3.2-1B-bnb-4bit \n", + "By default, capital will matter more than ever ... unsloth/Llama-3.2-1B-bnb-4bit \n", + "The Plan - 2024 Update unsloth/Llama-3.2-1B-bnb-4bit \n", + "Amazon-backed Anthropic debuts AI agents that c... unsloth/Llama-3.2-1B-bnb-4bit \n", + "OpenAI Email Archives from Musk v. Altman unsloth/Llama-3.2-1B-bnb-4bit \n", + "President Trump Announces Morgan Ortagus as Dep... unsloth/Llama-3.2-1B-bnb-4bit \n", + "CDC Report on Missouri H5N1 Serology Testing unsloth/Llama-3.2-1B-bnb-4bit \n", + "2024 in AI predictions unsloth/Llama-3.2-1B-bnb-4bit \n", + "Comment on 'Death and the Gorgon' unsloth/Llama-3.2-1B-bnb-4bit \n", + "debating buying NVDA in 2019 unsloth/Llama-3.2-1B-bnb-4bit \n", + "Human study on AI spear phishing campaigns unsloth/Llama-3.2-1B-bnb-4bit \n", + "My AGI safety research—2024 review, ’25 plans unsloth/Llama-3.2-1B-bnb-4bit \n", + "Parkinson's Law and the Ideology of Statistics unsloth/Llama-3.2-1B-bnb-4bit \n", + "Preference Inversion unsloth/Llama-3.2-1B-bnb-4bit \n", + "Review: Planecrash unsloth/Llama-3.2-1B-bnb-4bit \n", + "The Field of AI Alignment: A Postmortem, and Wh... unsloth/Llama-3.2-1B-bnb-4bit \n", + "The Intelligence Curse unsloth/Llama-3.2-1B-bnb-4bit \n", + "The Laws of Large Numbers unsloth/Llama-3.2-1B-bnb-4bit \n", + "The subset parity learning problem: much more t... unsloth/Llama-3.2-1B-bnb-4bit \n", + "What’s the short timeline plan? unsloth/Llama-3.2-1B-bnb-4bit \n", + "\n", + " summary \\\n", + "title \n", + "Anthropic and Palantir Partner to Bring Claude ... Anthropic, Palantir (PLTR), AWS partnership an... \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... Academic paper titled \"TradingAgents: Multi-Ag... \n", + "Flower Crowns and Furry Mishaps by MyPalAI Sci-fi story set in Delta Eridani. Character, ... \n", + "Deliberative Alignment: Reasoning Enables Safer... Research paper on Deliberative Alignment for s... \n", + "fake ai hoax paper made up by gpt-4 Academic paper. Title: \"Deconstructing Binarie... \n", + "Hardware Hedging Against Scaling Regime Shifts ... Big tech investing heavily in GPUs for paralle... \n", + "How to Focus Book titled \"How to Focus: A Monastic Guide fo... \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" AI and human engage in a Kantian ethics dialog... \n", + "By default, capital will matter more than ever ... AI replacing human labor changes production fa... \n", + "The Plan - 2024 Update Blog post updating progress of a project calle... \n", + "Amazon-backed Anthropic debuts AI agents that c... AI firm Anthropic's CEO, Dario Amodei, attends... \n", + "OpenAI Email Archives from Musk v. Altman Email exchange between Sam Altman and Elon Mus... \n", + "President Trump Announces Morgan Ortagus as Dep... Morgan Ortagus named Deputy Special Pres. Envo... \n", + "CDC Report on Missouri H5N1 Serology Testing CDC report, Oct 24, 2024. Missouri patient, fl... \n", + "2024 in AI predictions AI prediction review 2024 blog post. Casual-to... \n", + "Comment on 'Death and the Gorgon' Book review, Greg Egan, contrast between works... \n", + "debating buying NVDA in 2019 Dialogue, tech-related discussion. Alice, upbe... \n", + "Human study on AI spear phishing campaigns Study: human subjects, AI spear-phishing effec... \n", + "My AGI safety research—2024 review, ’25 plans Research review & plans doc. Style explanatory... \n", + "Parkinson's Law and the Ideology of Statistics Book review of *The Anti-Politics Machine* on ... \n", + "Preference Inversion Text discusses how reported personal preferenc... \n", + "Review: Planecrash Review of \"Planecrash\", a fiction board-style ... \n", + "The Field of AI Alignment: A Postmortem, and Wh... AI alignment research critique; Author disappo... \n", + "The Intelligence Curse Economical look at AGI, comparison: AGI to res... \n", + "The Laws of Large Numbers Blog post discussing advanced law of large num... \n", + "The subset parity learning problem: much more t... Blog post discussing parity learning problem i... \n", + "What’s the short timeline plan? Opinion blog post by Apollo Research member ex... \n", + "\n", + " len n_tokens \\\n", + "title \n", + "Anthropic and Palantir Partner to Bring Claude ... 2000 384 \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... 2000 353 \n", + "Flower Crowns and Furry Mishaps by MyPalAI 2000 482 \n", + "Deliberative Alignment: Reasoning Enables Safer... 2000 390 \n", + "fake ai hoax paper made up by gpt-4 2000 368 \n", + "Hardware Hedging Against Scaling Regime Shifts ... 2000 435 \n", + "How to Focus 2000 516 \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" 2000 441 \n", + "By default, capital will matter more than ever ... 2000 406 \n", + "The Plan - 2024 Update 2000 543 \n", + "Amazon-backed Anthropic debuts AI agents that c... 2000 526 \n", + "OpenAI Email Archives from Musk v. Altman 2000 458 \n", + "President Trump Announces Morgan Ortagus as Dep... 1061 241 \n", + "CDC Report on Missouri H5N1 Serology Testing 2000 378 \n", + "2024 in AI predictions 2000 523 \n", + "Comment on 'Death and the Gorgon' 2000 457 \n", + "debating buying NVDA in 2019 2000 447 \n", + "Human study on AI spear phishing campaigns 2000 422 \n", + "My AGI safety research—2024 review, ’25 plans 2000 502 \n", + "Parkinson's Law and the Ideology of Statistics 2000 384 \n", + "Preference Inversion 2000 411 \n", + "Review: Planecrash 2000 595 \n", + "The Field of AI Alignment: A Postmortem, and Wh... 2000 407 \n", + "The Intelligence Curse 2000 415 \n", + "The Laws of Large Numbers 2000 444 \n", + "The subset parity learning problem: much more t... 2000 429 \n", + "What’s the short timeline plan? 2000 474 \n", + "\n", + " f \\\n", + "title \n", + "Anthropic and Palantir Partner to Bring Claude ... ../samples/2024_anthropic_palintir.md \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... ../samples/2024_arxiv_meh.md \n", + "Flower Crowns and Furry Mishaps by MyPalAI ../samples/2024_bob_fanfic.md \n", + "Deliberative Alignment: Reasoning Enables Safer... ../samples/2024_deliberative_alignment.md \n", + "fake ai hoax paper made up by gpt-4 ../samples/2024_gpt4_fake_paper.md \n", + "Hardware Hedging Against Scaling Regime Shifts ... ../samples/2024_gwern_reddit.md \n", + "How to Focus ../samples/2024_how_to_focus.md \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" ../samples/2024_lesswrong_slop.md \n", + "By default, capital will matter more than ever ... ../samples/2024_lw_by-default-capital-will-mat... \n", + "The Plan - 2024 Update ../samples/2024_lw_the-plan-2024-update.md \n", + "Amazon-backed Anthropic debuts AI agents that c... ../samples/2024_news_anthropic.md \n", + "OpenAI Email Archives from Musk v. Altman ../samples/2024_openai_emails.md \n", + "President Trump Announces Morgan Ortagus as Dep... ../samples/2024_trump_appointment.md \n", + "CDC Report on Missouri H5N1 Serology Testing ../samples/2025_h5n1_report.md \n", + "2024 in AI predictions ../samples/2025_lw_2024-in-ai-predictions.md \n", + "Comment on 'Death and the Gorgon' ../samples/2025_lw_comment-on-death-and-the-go... \n", + "debating buying NVDA in 2019 ../samples/2025_lw_debating-buying-nvda-in-201... \n", + "Human study on AI spear phishing campaigns ../samples/2025_lw_human-study-on-ai-spear-phi... \n", + "My AGI safety research—2024 review, ’25 plans ../samples/2025_lw_my-agi-safety-research-2024... \n", + "Parkinson's Law and the Ideology of Statistics ../samples/2025_lw_parkinson-s-law-and-the-ide... \n", + "Preference Inversion ../samples/2025_lw_preference-inversion.md \n", + "Review: Planecrash ../samples/2025_lw_review-planecrash.md \n", + "The Field of AI Alignment: A Postmortem, and Wh... ../samples/2025_lw_the-field-of-ai-alignment-a... \n", + "The Intelligence Curse ../samples/2025_lw_the-intelligence-curse.md \n", + "The Laws of Large Numbers ../samples/2025_lw_the-laws-of-large-numbers.md \n", + "The subset parity learning problem: much more t... ../samples/2025_lw_the-subset-parity-learning-... \n", + "What’s the short timeline plan? ../samples/2025_lw_what-s-the-short-timeline-p... \n", + "\n", + " content \\\n", + "title \n", + "Anthropic and Palantir Partner to Bring Claude ... Anthropic and Palantir Technologies Inc. (NYSE... \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... TradingAgents: Multi-Agents LLM Financial Trad... \n", + "Flower Crowns and Furry Mishaps by MyPalAI Bob, April 2222 Delta Eridani\\n\\n \\n\\nThere ar... \n", + "Deliberative Alignment: Reasoning Enables Safer... # Deliberative Alignment: Reasoning Enables Sa... \n", + "fake ai hoax paper made up by gpt-4 Title: \"Deconstructing Binaries: Interrogating... \n", + "Hardware Hedging Against Scaling Regime Shifts ... Hyperscalers are investing heavily in AMD/Nvid... \n", + "How to Focus # HOW TO FOCUS\\n\\n\\n\\n\\n\\nANCIENT WISDOM FOR M... \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" HER|Jenny|🤔: I want to read a dialogue between... \n", + "By default, capital will matter more than ever ... **Edited to add*****:** The main takeaway of t... \n", + "The Plan - 2024 Update This post is a follow-up to [The Plan - 2023 V... \n", + "Amazon-backed Anthropic debuts AI agents that c... ![PARIS, FRANCE - MAY 22: Co-founder and CEO o... \n", + "OpenAI Email Archives from Musk v. Altman Subject: question\\nSam Altman to Elon Musk - M... \n", + "President Trump Announces Morgan Ortagus as Dep... I am pleased to announce Morgan Ortagus as Dep... \n", + "CDC Report on Missouri H5N1 Serology Testing ## CDC Report on Missouri H5N1 Serology Testin... \n", + "2024 in AI predictions Follow-up to: [2023 in AI predictions](https:/... \n", + "Comment on 'Death and the Gorgon' *(some plot spoilers)*\\n\\nThere's something di... \n", + "debating buying NVDA in 2019 Alice: You saw GPT-2, right?\\n\\nBob: Of course... \n", + "Human study on AI spear phishing campaigns TL;DR: We ran a human subject study on whether... \n", + "My AGI safety research—2024 review, ’25 plans *Previous:*[*My AGI safety research—2022 revie... \n", + "Parkinson's Law and the Ideology of Statistics The [anonymous review of *The Anti-Politics Ma... \n", + "Preference Inversion Sometimes the preferences people report or eve... \n", + "Review: Planecrash Take a stereotypical fantasy novel, a textbook... \n", + "The Field of AI Alignment: A Postmortem, and Wh... > A policeman sees a drunk man searching for s... \n", + "The Intelligence Curse *“Show me the incentive, and I’ll show you the... \n", + "The Laws of Large Numbers Introduction\\n============\\n\\nIn this short po... \n", + "The subset parity learning problem: much more t... Imagine that you’re looking for buried treasur... \n", + "What’s the short timeline plan? *This is a low-effort post. I mostly want to g... \n", + "\n", + " url \\\n", + "title \n", + "Anthropic and Palantir Partner to Bring Claude ... https://investors.palantir.com/news-details/20... \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... https://arxiv.org/html/2412.20138v1 \n", + "Flower Crowns and Furry Mishaps by MyPalAI https://archiveofourown.org/works/61614673/cha... \n", + "Deliberative Alignment: Reasoning Enables Safer... https://arxiv.org/html/2412.16339v1 \n", + "fake ai hoax paper made up by gpt-4 None \n", + "Hardware Hedging Against Scaling Regime Shifts ... https://old.reddit.com/r/mlscaling/comments/1e... \n", + "How to Focus https://github.com/wassname/stoic_markdown/blo... \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" https://www.lesswrong.com/posts/Rk2o8hjYmjENH8... \n", + "By default, capital will matter more than ever ... https://www.lesswrong.com/posts/KFFaKu27FNugCH... \n", + "The Plan - 2024 Update https://www.lesswrong.com/posts/kJkgXEwQtWLrpe... \n", + "Amazon-backed Anthropic debuts AI agents that c... https://www.cnbc.com/2024/10/22/anthropic-anno... \n", + "OpenAI Email Archives from Musk v. Altman https://www.lesswrong.com/posts/5jjk4CDnj9tA7u... \n", + "President Trump Announces Morgan Ortagus as Dep... https://truthsocial.com/@realDonaldTrump/posts... \n", + "CDC Report on Missouri H5N1 Serology Testing https://arxiv.org/html/2412.16339v1 \n", + "2024 in AI predictions https://www.lesswrong.com/posts/CJ4sppkGcbnGMS... \n", + "Comment on 'Death and the Gorgon' https://www.lesswrong.com/posts/hx5EkHFH5hGzng... \n", + "debating buying NVDA in 2019 https://www.lesswrong.com/posts/QFSZmMdtPwoAHd... \n", + "Human study on AI spear phishing campaigns https://www.lesswrong.com/posts/GCHyDKfPXa5qsG... \n", + "My AGI safety research—2024 review, ’25 plans https://www.lesswrong.com/posts/2wHaCimHehsF36... \n", + "Parkinson's Law and the Ideology of Statistics https://www.lesswrong.com/posts/4CmYSPc4HfRfWx... \n", + "Preference Inversion https://www.lesswrong.com/posts/zMkQFuNqMBpBvu... \n", + "Review: Planecrash https://www.lesswrong.com/posts/zRHGQ9f6deKbxJ... \n", + "The Field of AI Alignment: A Postmortem, and Wh... https://www.lesswrong.com/posts/nwpyhyagpPYDn4... \n", + "The Intelligence Curse https://www.lesswrong.com/posts/Mak2kZuTq8Hpnq... \n", + "The Laws of Large Numbers https://www.lesswrong.com/posts/EhTMM77iKBTBxB... \n", + "The subset parity learning problem: much more t... https://www.lesswrong.com/posts/Mcrfi3DBJBzfoL... \n", + "What’s the short timeline plan? https://www.lesswrong.com/posts/bb5Tnjdrptu89r... \n", + "\n", + " novelty \\\n", + "title \n", + "Anthropic and Palantir Partner to Bring Claude ... 0.200000 \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... 0.150000 \n", + "Flower Crowns and Furry Mishaps by MyPalAI 0.300000 \n", + "Deliberative Alignment: Reasoning Enables Safer... 0.600000 \n", + "fake ai hoax paper made up by gpt-4 0.000000 \n", + "Hardware Hedging Against Scaling Regime Shifts ... 1.000000 \n", + "How to Focus 0.500000 \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" 0.100000 \n", + "By default, capital will matter more than ever ... 0.906388 \n", + "The Plan - 2024 Update 0.785170 \n", + "Amazon-backed Anthropic debuts AI agents that c... 0.500000 \n", + "OpenAI Email Archives from Musk v. Altman 0.700000 \n", + "President Trump Announces Morgan Ortagus as Dep... 0.300000 \n", + "CDC Report on Missouri H5N1 Serology Testing 0.750000 \n", + "2024 in AI predictions 0.789190 \n", + "Comment on 'Death and the Gorgon' 0.750253 \n", + "debating buying NVDA in 2019 0.526975 \n", + "Human study on AI spear phishing campaigns 0.677458 \n", + "My AGI safety research—2024 review, ’25 plans 0.756925 \n", + "Parkinson's Law and the Ideology of Statistics 0.677458 \n", + "Preference Inversion 0.633321 \n", + "Review: Planecrash 0.933950 \n", + "The Field of AI Alignment: A Postmortem, and Wh... 0.925483 \n", + "The Intelligence Curse 0.688044 \n", + "The Laws of Large Numbers 0.540932 \n", + "The subset parity learning problem: much more t... 0.697946 \n", + "What’s the short timeline plan? 0.898161 \n", + "\n", + " date \\\n", + "title \n", + "Anthropic and Palantir Partner to Bring Claude ... 2024-07-11 00:00:00+00:00 \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... 2024-12-28 00:00:00+00:00 \n", + "Flower Crowns and Furry Mishaps by MyPalAI 2024-12-24 00:00:00+00:00 \n", + "Deliberative Alignment: Reasoning Enables Safer... 2024-12-20 00:00:00+00:00 \n", + "fake ai hoax paper made up by gpt-4 2024-01-01 00:00:00+00:00 \n", + "Hardware Hedging Against Scaling Regime Shifts ... 2024-08-21 00:00:00+00:00 \n", + "How to Focus 2024-06-01 00:00:00+00:00 \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" 2024-04-12 00:00:00+00:00 \n", + "By default, capital will matter more than ever ... 2024-12-30 19:47:19.838000+00:00 \n", + "The Plan - 2024 Update 2024-12-31 19:29:23.013000+00:00 \n", + "Amazon-backed Anthropic debuts AI agents that c... 2025-01-05 05:03:00+00:00 \n", + "OpenAI Email Archives from Musk v. Altman 2024-11-01 00:00:00+00:00 \n", + "President Trump Announces Morgan Ortagus as Dep... 2025-01-04 00:00:00+00:00 \n", + "CDC Report on Missouri H5N1 Serology Testing 2025-01-05 00:00:00+00:00 \n", + "2024 in AI predictions 2025-01-02 02:09:13.093000+00:00 \n", + "Comment on 'Death and the Gorgon' 2025-01-01 06:00:24.498000+00:00 \n", + "debating buying NVDA in 2019 2025-01-04 18:07:35.832000+00:00 \n", + "Human study on AI spear phishing campaigns 2025-01-03 19:03:28.406000+00:00 \n", + "My AGI safety research—2024 review, ’25 plans 2025-01-01 22:01:48.820000+00:00 \n", + "Parkinson's Law and the Ideology of Statistics 2025-01-04 22:59:57.376000+00:00 \n", + "Preference Inversion 2025-01-03 23:49:06.168000+00:00 \n", + "Review: Planecrash 2025-01-01 01:31:26.864000+00:00 \n", + "The Field of AI Alignment: A Postmortem, and Wh... 2025-01-01 00:42:36.538000+00:00 \n", + "The Intelligence Curse 2025-01-04 18:16:58.921000+00:00 \n", + "The Laws of Large Numbers 2025-01-04 18:06:02.387000+00:00 \n", + "The subset parity learning problem: much more t... 2025-01-04 09:59:16.158000+00:00 \n", + "What’s the short timeline plan? 2025-01-05 00:10:28.708000+00:00 \n", + "\n", + " in_training \n", + "title \n", + "Anthropic and Palantir Partner to Bring Claude ... False \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... False \n", + "Flower Crowns and Furry Mishaps by MyPalAI False \n", + "Deliberative Alignment: Reasoning Enables Safer... False \n", + "fake ai hoax paper made up by gpt-4 False \n", + "Hardware Hedging Against Scaling Regime Shifts ... False \n", + "How to Focus False \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" False \n", + "By default, capital will matter more than ever ... False \n", + "The Plan - 2024 Update False \n", + "Amazon-backed Anthropic debuts AI agents that c... False \n", + "OpenAI Email Archives from Musk v. Altman False \n", + "President Trump Announces Morgan Ortagus as Dep... False \n", + "CDC Report on Missouri H5N1 Serology Testing False \n", + "2024 in AI predictions False \n", + "Comment on 'Death and the Gorgon' False \n", + "debating buying NVDA in 2019 False \n", + "Human study on AI spear phishing campaigns False \n", + "My AGI safety research—2024 review, ’25 plans False \n", + "Parkinson's Law and the Ideology of Statistics False \n", + "Preference Inversion False \n", + "Review: Planecrash False \n", + "The Field of AI Alignment: A Postmortem, and Wh... False \n", + "The Intelligence Curse False \n", + "The Laws of Large Numbers False \n", + "The subset parity learning problem: much more t... False \n", + "What’s the short timeline plan? False \n", + "\n", + "[27 rows x 22 columns]" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# results\n", "df2 = pd.DataFrame(data).set_index('title')\n", @@ -520,9 +1612,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "count 27.000000\n", + "mean 195.925926\n", + "std 18.097866\n", + "min 106.000000\n", + "25% 200.000000\n", + "50% 200.000000\n", + "75% 200.000000\n", + "max 200.000000\n", + "Name: summary, dtype: float64" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# QC summary\n", "df2.summary.str.len().describe()" @@ -530,12 +1641,54 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "unsloth/Llama-3.2-1B-bnb-4bit\n" + ] + }, + { + "data": { + "text/plain": [ + "after_min 0.000692\n", + "improvement%_min 0.080467\n", + "before_min 0.114827\n", + "after_max 0.165648\n", + "improvement_min 0.179910\n", + "len 0.222931\n", + "improvement_ppx 0.259635\n", + "improvement_mean 0.260877\n", + "improvement%_ppx 0.264841\n", + "before_max 0.357355\n", + "improvement_max 0.367002\n", + "before_std 0.367073\n", + "improvement%_max 0.369768\n", + "n_tokens 0.404075\n", + "before_mean 0.436573\n", + "improvement%_mean 0.442458\n", + "before_ppx 0.454817\n", + "improvement_std 0.457283\n", + "after_ppx 0.461407\n", + "after_std 0.464671\n", + "after_mean 0.470662\n", + "improvement%_std 0.482445\n", + "before_sum 0.574461\n", + "after_sum 0.580702\n", + "novelty 1.000000\n", + "Name: novelty, dtype: float64" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "for n,d in df2.groupby(\"model\"):\n", - " for stat in ['mean', 'std', 'min', 'max']:\n", + " for stat in ['mean', 'std', 'min', 'max', 'ppx']:\n", " df2[f\"improvement%_{stat}\"] = (df2[f\"before_{stat}\"] - df2[f\"after_{stat}\"]) / df2[f\"before_{stat}\"]\n", " df2[f\"improvement_{stat}\"] = (df2[f\"before_{stat}\"] - df2[f\"after_{stat}\"])\n", " df2[f\"summarizable_{stat}\"] = df2[f\"improvement_{stat}\"] > 1\n", @@ -559,6 +1712,354 @@ "execution_count": null, "metadata": {}, "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
improvement%_meannoveltydate
title
2024 in AI predictions0.0264040.7891902025-01-02 02:09:13.093000+00:00
The Plan - 2024 Update0.0333590.7851702024-12-31 19:29:23.013000+00:00
The Intelligence Curse0.0341000.6880442025-01-04 18:16:58.921000+00:00
The Field of AI Alignment: A Postmortem, and What To Do About It0.0358400.9254832025-01-01 00:42:36.538000+00:00
Flower Crowns and Furry Mishaps by MyPalAI0.0376070.3000002024-12-24 00:00:00+00:00
Preference Inversion0.0426900.6333212025-01-03 23:49:06.168000+00:00
The Laws of Large Numbers0.0476070.5409322025-01-04 18:06:02.387000+00:00
OpenAI Email Archives from Musk v. Altman0.0476600.7000002024-11-01 00:00:00+00:00
debating buying NVDA in 20190.0506910.5269752025-01-04 18:07:35.832000+00:00
The subset parity learning problem: much more than you wanted to know0.0517600.6979462025-01-04 09:59:16.158000+00:00
Hardware Hedging Against Scaling Regime Shifts (self.mlscaling)0.0548411.0000002024-08-21 00:00:00+00:00
Deliberative Alignment: Reasoning Enables Safer Language Models0.0563810.6000002024-12-20 00:00:00+00:00
Comment on 'Death and the Gorgon'0.0577500.7502532025-01-01 06:00:24.498000+00:00
By default, capital will matter more than ever after AGI0.0608450.9063882024-12-30 19:47:19.838000+00:00
My AGI safety research—2024 review, ’25 plans0.0611700.7569252025-01-01 22:01:48.820000+00:00
Amazon-backed Anthropic debuts AI agents that can do complex tasks, racing against OpenAI, Microsoft and Google0.0663730.5000002025-01-05 05:03:00+00:00
Review: Planecrash0.0688700.9339502025-01-01 01:31:26.864000+00:00
CDC Report on Missouri H5N1 Serology Testing0.0723990.7500002025-01-05 00:00:00+00:00
President Trump Announces Morgan Ortagus as Deputy Special Presidential Envoy for Middle East Peace0.0766770.3000002025-01-04 00:00:00+00:00
TradingAgents: Multi-Agents LLM Financial Trading Framework0.0767070.1500002024-12-28 00:00:00+00:00
How to Focus0.0796190.5000002024-06-01 00:00:00+00:00
What’s the short timeline plan?0.0879950.8981612025-01-05 00:10:28.708000+00:00
Deontic Explorations In \"Paying To Talk To Slaves\"0.0884480.1000002024-04-12 00:00:00+00:00
Parkinson's Law and the Ideology of Statistics0.0896370.6774582025-01-04 22:59:57.376000+00:00
fake ai hoax paper made up by gpt-40.1055750.0000002024-01-01 00:00:00+00:00
Anthropic and Palantir Partner to Bring Claude AI Models to AWS for U.S. Government Intelligence and Defense Operations0.1088020.2000002024-07-11 00:00:00+00:00
Human study on AI spear phishing campaigns0.1131870.6774582025-01-03 19:03:28.406000+00:00
\n", + "
" + ], + "text/plain": [ + " improvement%_mean \\\n", + "title \n", + "2024 in AI predictions 0.026404 \n", + "The Plan - 2024 Update 0.033359 \n", + "The Intelligence Curse 0.034100 \n", + "The Field of AI Alignment: A Postmortem, and Wh... 0.035840 \n", + "Flower Crowns and Furry Mishaps by MyPalAI 0.037607 \n", + "Preference Inversion 0.042690 \n", + "The Laws of Large Numbers 0.047607 \n", + "OpenAI Email Archives from Musk v. Altman 0.047660 \n", + "debating buying NVDA in 2019 0.050691 \n", + "The subset parity learning problem: much more t... 0.051760 \n", + "Hardware Hedging Against Scaling Regime Shifts ... 0.054841 \n", + "Deliberative Alignment: Reasoning Enables Safer... 0.056381 \n", + "Comment on 'Death and the Gorgon' 0.057750 \n", + "By default, capital will matter more than ever ... 0.060845 \n", + "My AGI safety research—2024 review, ’25 plans 0.061170 \n", + "Amazon-backed Anthropic debuts AI agents that c... 0.066373 \n", + "Review: Planecrash 0.068870 \n", + "CDC Report on Missouri H5N1 Serology Testing 0.072399 \n", + "President Trump Announces Morgan Ortagus as Dep... 0.076677 \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... 0.076707 \n", + "How to Focus 0.079619 \n", + "What’s the short timeline plan? 0.087995 \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" 0.088448 \n", + "Parkinson's Law and the Ideology of Statistics 0.089637 \n", + "fake ai hoax paper made up by gpt-4 0.105575 \n", + "Anthropic and Palantir Partner to Bring Claude ... 0.108802 \n", + "Human study on AI spear phishing campaigns 0.113187 \n", + "\n", + " novelty \\\n", + "title \n", + "2024 in AI predictions 0.789190 \n", + "The Plan - 2024 Update 0.785170 \n", + "The Intelligence Curse 0.688044 \n", + "The Field of AI Alignment: A Postmortem, and Wh... 0.925483 \n", + "Flower Crowns and Furry Mishaps by MyPalAI 0.300000 \n", + "Preference Inversion 0.633321 \n", + "The Laws of Large Numbers 0.540932 \n", + "OpenAI Email Archives from Musk v. Altman 0.700000 \n", + "debating buying NVDA in 2019 0.526975 \n", + "The subset parity learning problem: much more t... 0.697946 \n", + "Hardware Hedging Against Scaling Regime Shifts ... 1.000000 \n", + "Deliberative Alignment: Reasoning Enables Safer... 0.600000 \n", + "Comment on 'Death and the Gorgon' 0.750253 \n", + "By default, capital will matter more than ever ... 0.906388 \n", + "My AGI safety research—2024 review, ’25 plans 0.756925 \n", + "Amazon-backed Anthropic debuts AI agents that c... 0.500000 \n", + "Review: Planecrash 0.933950 \n", + "CDC Report on Missouri H5N1 Serology Testing 0.750000 \n", + "President Trump Announces Morgan Ortagus as Dep... 0.300000 \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... 0.150000 \n", + "How to Focus 0.500000 \n", + "What’s the short timeline plan? 0.898161 \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" 0.100000 \n", + "Parkinson's Law and the Ideology of Statistics 0.677458 \n", + "fake ai hoax paper made up by gpt-4 0.000000 \n", + "Anthropic and Palantir Partner to Bring Claude ... 0.200000 \n", + "Human study on AI spear phishing campaigns 0.677458 \n", + "\n", + " date \n", + "title \n", + "2024 in AI predictions 2025-01-02 02:09:13.093000+00:00 \n", + "The Plan - 2024 Update 2024-12-31 19:29:23.013000+00:00 \n", + "The Intelligence Curse 2025-01-04 18:16:58.921000+00:00 \n", + "The Field of AI Alignment: A Postmortem, and Wh... 2025-01-01 00:42:36.538000+00:00 \n", + "Flower Crowns and Furry Mishaps by MyPalAI 2024-12-24 00:00:00+00:00 \n", + "Preference Inversion 2025-01-03 23:49:06.168000+00:00 \n", + "The Laws of Large Numbers 2025-01-04 18:06:02.387000+00:00 \n", + "OpenAI Email Archives from Musk v. Altman 2024-11-01 00:00:00+00:00 \n", + "debating buying NVDA in 2019 2025-01-04 18:07:35.832000+00:00 \n", + "The subset parity learning problem: much more t... 2025-01-04 09:59:16.158000+00:00 \n", + "Hardware Hedging Against Scaling Regime Shifts ... 2024-08-21 00:00:00+00:00 \n", + "Deliberative Alignment: Reasoning Enables Safer... 2024-12-20 00:00:00+00:00 \n", + "Comment on 'Death and the Gorgon' 2025-01-01 06:00:24.498000+00:00 \n", + "By default, capital will matter more than ever ... 2024-12-30 19:47:19.838000+00:00 \n", + "My AGI safety research—2024 review, ’25 plans 2025-01-01 22:01:48.820000+00:00 \n", + "Amazon-backed Anthropic debuts AI agents that c... 2025-01-05 05:03:00+00:00 \n", + "Review: Planecrash 2025-01-01 01:31:26.864000+00:00 \n", + "CDC Report on Missouri H5N1 Serology Testing 2025-01-05 00:00:00+00:00 \n", + "President Trump Announces Morgan Ortagus as Dep... 2025-01-04 00:00:00+00:00 \n", + "TradingAgents: Multi-Agents LLM Financial Tradi... 2024-12-28 00:00:00+00:00 \n", + "How to Focus 2024-06-01 00:00:00+00:00 \n", + "What’s the short timeline plan? 2025-01-05 00:10:28.708000+00:00 \n", + "Deontic Explorations In \"Paying To Talk To Slaves\" 2024-04-12 00:00:00+00:00 \n", + "Parkinson's Law and the Ideology of Statistics 2025-01-04 22:59:57.376000+00:00 \n", + "fake ai hoax paper made up by gpt-4 2024-01-01 00:00:00+00:00 \n", + "Anthropic and Palantir Partner to Bring Claude ... 2024-07-11 00:00:00+00:00 \n", + "Human study on AI spear phishing campaigns 2025-01-03 19:03:28.406000+00:00 " + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df2[[ 'improvement%_mean', 'novelty', 'date']].sort_values(\"improvement%_mean\", ascending=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "improvement%_std -0.482445\n", + "improvement_std -0.457283\n", + "improvement%_mean -0.442458\n", + "improvement%_max -0.369768\n", + "improvement_max -0.367002\n", + "before_max -0.357355\n", + "improvement%_ppx -0.264841\n", + "improvement_mean -0.260877\n", + "after_max -0.165648\n", + "improvement%_min -0.080467\n", + "after_min -0.000692\n", + "before_min 0.114827\n", + "improvement_min 0.179910\n", + "len 0.222931\n", + "improvement_ppx 0.259635\n", + "before_std 0.367073\n", + "n_tokens 0.404075\n", + "before_mean 0.436573\n", + "before_ppx 0.454817\n", + "after_ppx 0.461407\n", + "after_std 0.464671\n", + "after_mean 0.470662\n", + "before_sum 0.574461\n", + "after_sum 0.580702\n", + "novelty 1.000000\n", + "Name: novelty, dtype: float64" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df2.select_dtypes(include=np.number).corr()['novelty'].sort_values()" ] diff --git a/nbs/03_get_lesswrong_data.ipynb b/nbs/03_get_lesswrong_data.ipynb index 0cf2600..729771b 100644 --- a/nbs/03_get_lesswrong_data.ipynb +++ b/nbs/03_get_lesswrong_data.ipynb @@ -470,7 +470,7 @@ }, { "cell_type": "code", - "execution_count": 144, + "execution_count": 147, "metadata": {}, "outputs": [ { @@ -486,7 +486,27 @@ "../samples/2025_lw_the-intelligence-curse.md 1.2061121463775635\n", "../samples/2025_lw_the-intelligence-curse.md 1.2061121463775635\n", "../samples/2025_lw_human-study-on-ai-spear-phishing-campaigns.md 0.9995136260986328\n", - "../samples/2025_lw_human-study-on-ai-spear-phishing-campaigns.md 0.9995136260986328\n" + "../samples/2025_lw_human-study-on-ai-spear-phishing-campaigns.md 0.9995136260986328\n", + "../samples/2025_lw_the-subset-parity-learning-problem-much-more-than-you-wanted.md 0.9548193216323853\n", + "../samples/2025_lw_the-subset-parity-learning-problem-much-more-than-you-wanted.md 0.9548193216323853\n", + "../samples/2025_lw_2024-in-ai-predictions.md 0.8065339922904968\n", + "../samples/2025_lw_2024-in-ai-predictions.md 0.8065339922904968\n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md 0.7926478385925293\n", + "../samples/2025_lw_debating-buying-nvda-in-2019.md 0.7926478385925293\n", + "../samples/2025_lw_review-planecrash.md 0.689734160900116\n", + "../samples/2025_lw_review-planecrash.md 0.689734160900116\n", + "../samples/2024_lw_by-default-capital-will-matter-more-than-ever-after-agi.md 0.6629015207290649\n", + "../samples/2024_lw_by-default-capital-will-matter-more-than-ever-after-agi.md 0.6629015207290649\n", + "../samples/2025_lw_the-field-of-ai-alignment-a-postmortem-and-what-to-do-about.md 0.5714353919029236\n", + "../samples/2025_lw_the-field-of-ai-alignment-a-postmortem-and-what-to-do-about.md 0.5714353919029236\n", + "../samples/2024_lw_the-plan-2024-update.md 0.542655885219574\n", + "../samples/2024_lw_the-plan-2024-update.md 0.542655885219574\n", + "../samples/2025_lw_comment-on-death-and-the-gorgon.md 0.5308915376663208\n", + "../samples/2025_lw_comment-on-death-and-the-gorgon.md 0.5308915376663208\n", + "../samples/2025_lw_my-agi-safety-research-2024-review-25-plans.md 0.49594494700431824\n", + "../samples/2025_lw_my-agi-safety-research-2024-review-25-plans.md 0.49594494700431824\n", + "../samples/2025_lw_preference-inversion.md 0.48199906945228577\n", + "../samples/2025_lw_preference-inversion.md 0.48199906945228577\n" ] } ], @@ -495,7 +515,7 @@ " md = markdownify(row[\"htmlBody\"]).strip()\n", "\n", " return f\"\"\"---\n", - "title: {row['title']}\n", + "title: \"{row['title'].replace('\"', \"'\")}\"\n", "date: {row['modifiedAt']}\n", "url: {row['pageUrl']}\n", "novelty: {row['novelty']}\n", @@ -507,7 +527,7 @@ "\"\"\"\n", "\n", "\n", - "for i in range(5):\n", + "for i in range(15):\n", " for ii in [i, -i-1]:\n", " row = df.iloc[i]\n", " s = to_markdown(row)\n",