{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://github.com/huggingface/peft/blob/main/examples/fp4_finetuning/finetune_fp4_opt_bnb_peft.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import optim\n",
    "import lightning as pl\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from loguru import logger\n",
    "import sys\n",
    "\n",
    "# only if you want it shorter\n",
    "logger.remove()\n",
    "logger.add(sys.stderr, format=\"<level>{message}</level>\", level=\"WARNING\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES']=\"1\"\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import transformers\n",
    "from datasets import load_dataset\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, AutoConfig\n",
    "import numpy as np\n",
    "from tqdm.auto import tqdm\n",
    "import pandas as pd\n",
    "import warnings\n",
    "from peft import LoraConfig, get_peft_model, IA3Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.style.use('seaborn-v0_8')\n",
    "torch.set_float32_matmul_precision('medium')\n",
    "warnings.filterwarnings(\"ignore\", \".*does not have many workers.*\")\n",
    "warnings.filterwarnings(\"ignore\", \".*Was asked to gather along dimension 0.*\")\n",
    "warnings.filterwarnings(\"ignore\", \".*There is an imbalance between your GPUs.*\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_chars = 2000\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# https://huggingface.co/collections/unsloth/llama-32-66f46afde4ca573864321a22\n",
    "model_name = \"unsloth/Llama-3.2-1B\"\n",
    "model_name = \"unsloth/Llama-3.2-1B-bnb-4bit\"\n",
    "# Model Release Date: Sept 25, 2024\n",
    "# launch date 9/25/2024 https://github.com/meta-llama/llama-models/blob/main/README.md\n",
    "# https://colab.research.google.com/drive/1T5-zKWM_5OD21QHwXHiV9ixTRR7k3iB9?usp=sharing\n",
    "# unsloth/Llama-3.2-3B\n",
    "# Data Freshness: The pretraining data has a cutoff of December 2023.\n",
    "\n",
    "def load_model():\n",
    "\n",
    "    model = AutoModelForCausalLM.from_pretrained(\n",
    "        model_name,\n",
    "        # quantization_config=BitsAndBytesConfig(\n",
    "        #     load_in_4bit=True,\n",
    "        #     llm_int8_threshold=6.0,\n",
    "        #     llm_int8_has_fp16_weight=False,\n",
    "        #     bnb_4bit_compute_dtype=torch.float16,\n",
    "        #     bnb_4bit_use_double_quant=True,\n",
    "        #     bnb_4bit_quant_type=\"nf4\",\n",
    "        # ),\n",
    "        torch_dtype=torch.float16,\n",
    "        trust_remote_code=True,\n",
    "    )\n",
    "\n",
    "\n",
    "    # config = AutoConfig.from_pretrained(model_name, trust_remote_code=True,)\n",
    "    # config.quantization_config['use_exllama'] = False\n",
    "    # config.quantization_config['disable_exllama'] = True\n",
    "    # model = AutoModelForCausalLM.from_pretrained(\n",
    "    #     model_name,\n",
    "    #     torch_dtype=torch.bfloat16,\n",
    "    #     trust_remote_code=True,\n",
    "    #     config=config,\n",
    "    # )\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unused kwargs: ['_load_in_4bit', '_load_in_8bit', 'quant_method']. These kwargs are not used in <class 'transformers.utils.quantization_config.BitsAndBytesConfig'>.\n",
      "`low_cpu_mem_usage` was None, now default to True since model is quantized.\n"
     ]
    }
   ],
   "source": [
    "base_model = load_model()\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True,)\n",
    "tokenizer.pad_token = tokenizer.eos_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reset_model(base_model):\n",
    "    # peft_config = LoraConfig(\n",
    "    #     # task_type=TaskType.TOKEN_CLS, \n",
    "    #     target_modules=[ \"fc2\",  \"Wqkv\",],\n",
    "    #     inference_mode=False, r=4, lora_alpha=4, \n",
    "    #     # lora_dropout=0.1, \n",
    "    #     # bias=\"all\"\n",
    "    # )\n",
    "    # peft_config = IA3Config(\n",
    "    #     target_modules=[ \"fc2\",  \"Wqkv\",], \n",
    "    #         feedforward_modules=[\"fc2\"],\n",
    "    #         inference_mode=False,\n",
    "    # )\n",
    "    peft_config = IA3Config(\n",
    "        # target_modules=[ \"fc2\",  \"Wqkv\", 'out_proj', 'fc1'], \n",
    "        #     feedforward_modules=[\"fc2\", 'fc1', 'out_proj'],\n",
    "        #     inference_mode=False,\n",
    "    )\n",
    "    random_name = \"peft_\" + str(np.random.randint(0, 100000))\n",
    "    model = get_peft_model(base_model, peft_config, adapter_name=random_name)\n",
    "    model.config.use_cache = False\n",
    "    return model\n",
    "\n",
    "model = reset_model(base_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs_writing_detector.data.load_md import load_md_df\n",
    "from bs_writing_detector.metrics.ppx import perplexity_compute_ds\n",
    "from pathlib import Path\n",
    "df = load_md_df(Path(\"../samples/\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.nn import functional as F\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "from datasets import Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lightning helpers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval(model, tokenizer, ds_val: Dataset):\n",
    "    model.eval();\n",
    "    with torch.no_grad():\n",
    "        with model.disable_adapter():\n",
    "            results = perplexity_compute_ds(ds=ds_val, model=model, tokenizer=tokenizer, device='cuda')['nlls'][0]\n",
    "        results2 = perplexity_compute_ds(ds=ds_val, model=model, tokenizer=tokenizer, device='cuda')['nlls'][0]\n",
    "    return dict(before=results, after=results2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Dataset\n",
    "\n",
    "\n",
    "def compute_metrics(eval_prediction):\n",
    "    return {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Trainer docs\n",
    "\n",
    "- https://huggingface.co/docs/transformers/v4.36.1/en/main_classes/trainer#transformers.Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['input_ids', 'attention_mask', 'overflow_to_sample_mapping'],\n",
       "    num_rows: 2\n",
       "})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "def tokenize_and_split(examples):\n",
    "    l = len(tokenizer(examples).input_ids[0])\n",
    "    max_len = min(l//3, max_chars) # break into at least 5\n",
    "    max_len = max(max_len, 10)\n",
    "\n",
    "\n",
    "    result = tokenizer(\n",
    "        examples,\n",
    "        add_special_tokens=False,\n",
    "        truncation=True,\n",
    "        stride=2, # – If set to a number along with max_length, the overflowing tokens returned will contain some tokens from the main sequence returned. The value of this argument defines the number of additional tokens.\n",
    "        max_length=max_len,\n",
    "        return_overflowing_tokens=True,\n",
    "        return_attention_mask=True,\n",
    "    )\n",
    "    return result\n",
    "\n",
    "sample = df.sample(1).iloc[0]\n",
    "s = sample['content']\n",
    "d = Dataset.from_dict(tokenize_and_split([s]))\n",
    "d2  = d.train_test_split(test_size=0.5, seed=42)\n",
    "ds_train = d2['train']\n",
    "ds_val = d2['test']\n",
    "ds_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def learn_sample(sample):\n",
    "    # device = 'cuda'\n",
    "    # lr = 4e-3\n",
    "    # epochs = 3\n",
    "    # accum_steps = 1\n",
    "    batch_size = 1\n",
    "    verbose = False\n",
    "\n",
    "    s = sample['content']\n",
    "\n",
    "    d = Dataset.from_dict(tokenize_and_split([s]))\n",
    "    d2  = d.train_test_split(test_size=0.5, seed=42)\n",
    "    ds_train = d2['train']\n",
    "    ds_val = d2['test']\n",
    "\n",
    "    print(model.peft_config)\n",
    "    model = reset_model(base_model)\n",
    "\n",
    "    # verify that we have reset it\n",
    "    print(model.peft_config)\n",
    "\n",
    "\n",
    "    # eval(model, tokenizer, ds_train)\n",
    "\n",
    "    # https://huggingface.co/docs/transformers/v4.36.1/en/main_classes/trainer#transformers.Trainer\n",
    "    trainer = transformers.Trainer(\n",
    "        model=model,\n",
    "        train_dataset=ds_train,\n",
    "        eval_dataset=ds_val,\n",
    "        compute_metrics=compute_metrics, # without this it wont even give val loss\n",
    "        args=transformers.TrainingArguments(\n",
    "            # checkpoint='epoch',\n",
    "            save_strategy='epoch',\n",
    "            label_names=['labels',],\n",
    "            per_device_train_batch_size=batch_size,\n",
    "            # gradient_accumulation_steps=1,\n",
    "            # warmup_steps=6,\n",
    "            warmup_ratio=0.1,\n",
    "            # max_steps=50,\n",
    "            num_train_epochs=3,\n",
    "            learning_rate=1e-3,\n",
    "            fp16=True,\n",
    "            logging_steps=1,\n",
    "            output_dir=\"outputs\",\n",
    "            log_level='error',\n",
    "            # do_eval=True,\n",
    "            evaluation_strategy=\"epoch\",\n",
    "            eval_steps=1,\n",
    "            load_best_model_at_end=True,\n",
    "            \n",
    "            # disable_tqdm=not verbose,\n",
    "        ),\n",
    "        data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),\n",
    "    )\n",
    "    trainer._signature_columns = ['input_ids', 'attention_mask', 'labels',]\n",
    "    model.config.use_cache = False  # silence the warnings. Please re-enable for inference!\n",
    "    train_output = trainer.train()\n",
    "\n",
    "    df_hist = pd.DataFrame(trainer.state.log_history)\n",
    "    df_hist_epoch = df_hist.groupby('epoch').last().drop(columns=['step'])\n",
    "    df_hist_step = df_hist.set_index('step').dropna(thresh=2, axis=1)\n",
    "    if verbose:\n",
    "        df_hist_epoch['loss'].plot()\n",
    "        plt.twinx()\n",
    "        df_hist_epoch['eval_loss'].plot(c='b', label='eval')\n",
    "        plt.legend()\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "    result_train = {f'train/{k}':v for k,v in eval(model, tokenizer, ds_train).items()}\n",
    "    result = eval(model, tokenizer, ds_val)\n",
    "    result['hist'] = df_hist_epoch\n",
    "    result.update(result_train)\n",
    "    return result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['before', 'after', 'hist', 'train/before', 'train/after', 'title', 'f', 'content', 'url', 'novelty', 'date', 'in_training'])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[0].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46ca413f7112493bbeb2be6403b68e9f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/30 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.8911, 'grad_norm': 0.8076428174972534, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 2.3384, 'grad_norm': 0.6929064989089966, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.5387918949127197, 'eval_runtime': 0.1312, 'eval_samples_per_second': 15.24, 'eval_steps_per_second': 7.62, 'epoch': 1.0}\n",
      "{'loss': 2.7436, 'grad_norm': 0.7699084281921387, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.1441, 'grad_norm': 0.6051006317138672, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.5293452739715576, 'eval_runtime': 0.1328, 'eval_samples_per_second': 15.055, 'eval_steps_per_second': 7.528, 'epoch': 2.0}\n",
      "{'loss': 2.5154, 'grad_norm': 0.749704122543335, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 2.0386, 'grad_norm': 0.5838078856468201, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.5258588790893555, 'eval_runtime': 0.1318, 'eval_samples_per_second': 15.171, 'eval_steps_per_second': 7.585, 'epoch': 3.0}\n",
      "{'train_runtime': 1.3375, 'train_samples_per_second': 4.486, 'train_steps_per_second': 4.486, 'train_loss': 2.445200721422831, 'epoch': 3.0}\n",
      "Anthropic and Palantir Partner to Bring Claude AI Models to AWS for U.S. Government Intelligence and Defense Operations\n",
      "{'before': [11.264070510864258, 7.382840156555176, 12.170625686645508, 1.2329556941986084], 'after': [11.19461441040039, 7.479499816894531, 12.069757461547852, 0.9895272254943848]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.2976, 'grad_norm': 1.1653801202774048, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.499, 'grad_norm': 1.3143912553787231, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.3701250553131104, 'eval_runtime': 0.1588, 'eval_samples_per_second': 12.593, 'eval_steps_per_second': 6.297, 'epoch': 1.0}\n",
      "{'loss': 3.0943, 'grad_norm': 1.04639732837677, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 3.1374, 'grad_norm': 1.0442923307418823, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.351206064224243, 'eval_runtime': 0.127, 'eval_samples_per_second': 15.745, 'eval_steps_per_second': 7.872, 'epoch': 2.0}\n",
      "{'loss': 2.7867, 'grad_norm': 0.9014344811439514, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 2.9493, 'grad_norm': 0.9459804892539978, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.344069242477417, 'eval_runtime': 0.1281, 'eval_samples_per_second': 15.616, 'eval_steps_per_second': 7.808, 'epoch': 3.0}\n",
      "{'train_runtime': 1.3909, 'train_samples_per_second': 4.314, 'train_steps_per_second': 4.314, 'train_loss': 3.1273703972498574, 'epoch': 3.0}\n",
      "TradingAgents: Multi-Agents LLM Financial Trading Framework\n",
      "{'before': [10.053862571716309, 0.6636170744895935, 8.349092483520508, 1.452925443649292, 0.9928902387619019, 8.745048522949219], 'after': [9.95648193359375, 0.47155672311782837, 8.367955207824707, 1.2811455726623535, 0.875080943107605, 8.782474517822266]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.3063, 'grad_norm': 1.185476303100586, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.3628, 'grad_norm': 1.5031108856201172, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.6609110832214355, 'eval_runtime': 0.1593, 'eval_samples_per_second': 12.556, 'eval_steps_per_second': 6.278, 'epoch': 1.0}\n",
      "{'loss': 3.0717, 'grad_norm': 0.9321146011352539, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 3.0218, 'grad_norm': 1.1784974336624146, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.6481850147247314, 'eval_runtime': 0.1593, 'eval_samples_per_second': 12.552, 'eval_steps_per_second': 6.276, 'epoch': 2.0}\n",
      "{'loss': 2.7501, 'grad_norm': 0.7790453433990479, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 2.8441, 'grad_norm': 1.0859301090240479, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.6467812061309814, 'eval_runtime': 0.1589, 'eval_samples_per_second': 12.588, 'eval_steps_per_second': 6.294, 'epoch': 3.0}\n",
      "{'train_runtime': 1.6521, 'train_samples_per_second': 3.632, 'train_steps_per_second': 3.632, 'train_loss': 3.059459924697876, 'epoch': 3.0}\n",
      "Flower Crowns and Furry Mishaps by MyPalAI\n",
      "{'before': [7.296568393707275, 3.3268048763275146, 3.3179941177368164, 4.614715576171875, 4.787782669067383, 6.660912990570068], 'after': [7.300139904022217, 3.366487979888916, 3.34420108795166, 4.758848190307617, 4.731474876403809, 6.640608787536621]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 4.1433, 'grad_norm': 1.3135420083999634, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.8979, 'grad_norm': 1.6989140510559082, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.8832738399505615, 'eval_runtime': 0.1567, 'eval_samples_per_second': 12.767, 'eval_steps_per_second': 6.384, 'epoch': 1.0}\n",
      "{'loss': 3.8552, 'grad_norm': 1.1130166053771973, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 3.445, 'grad_norm': 1.2440484762191772, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.857309103012085, 'eval_runtime': 0.1665, 'eval_samples_per_second': 12.015, 'eval_steps_per_second': 6.007, 'epoch': 2.0}\n",
      "{'loss': 3.437, 'grad_norm': 1.004622459411621, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 3.2143, 'grad_norm': 1.0474002361297607, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.846127986907959, 'eval_runtime': 0.1571, 'eval_samples_per_second': 12.729, 'eval_steps_per_second': 6.365, 'epoch': 3.0}\n",
      "{'train_runtime': 2.0371, 'train_samples_per_second': 2.945, 'train_steps_per_second': 2.945, 'train_loss': 3.6654568910598755, 'epoch': 3.0}\n",
      "Paradox's Box (Bobiverse) by Mark4man\n",
      "{'before': [8.102655410766602, 4.443521976470947, 6.978631973266602, 3.8517813682556152, 6.285520076751709], 'after': [7.454442024230957, 4.559870719909668, 7.13922643661499, 4.0717997550964355, 6.418486595153809]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.5963, 'grad_norm': 1.7606483697891235, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.8611, 'grad_norm': nan, 'learning_rate': 0.001, 'epoch': 1.0}\n",
      "{'eval_loss': 4.20759391784668, 'eval_runtime': 0.1417, 'eval_samples_per_second': 14.112, 'eval_steps_per_second': 7.056, 'epoch': 1.0}\n",
      "{'loss': 3.5963, 'grad_norm': 1.7607665061950684, 'learning_rate': 0.0008, 'epoch': 1.5}\n",
      "{'loss': 3.8058, 'grad_norm': 5.883133411407471, 'learning_rate': 0.0006, 'epoch': 2.0}\n",
      "{'eval_loss': 4.105123519897461, 'eval_runtime': 0.1429, 'eval_samples_per_second': 13.994, 'eval_steps_per_second': 6.997, 'epoch': 2.0}\n",
      "{'loss': 2.6548, 'grad_norm': 1.1668059825897217, 'learning_rate': 0.0004, 'epoch': 2.5}\n",
      "{'loss': 3.4369, 'grad_norm': 1.433318853378296, 'learning_rate': 0.0002, 'epoch': 3.0}\n",
      "{'eval_loss': 4.070194721221924, 'eval_runtime': 0.1445, 'eval_samples_per_second': 13.837, 'eval_steps_per_second': 6.918, 'epoch': 3.0}\n",
      "{'train_runtime': 2.0652, 'train_samples_per_second': 2.905, 'train_steps_per_second': 2.905, 'train_loss': 3.4918715159098306, 'epoch': 3.0}\n",
      "Deliberative Alignment: Reasoning Enables Safer Language Models\n",
      "{'before': [11.071215629577637, 4.1971049308776855, 5.31649112701416, 0.9296282529830933], 'after': [10.824005126953125, 4.173127174377441, 5.1760783195495605, 1.0669198036193848]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.8892, 'grad_norm': 1.9384897947311401, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 2.5209, 'grad_norm': 1.7461295127868652, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 2.852980613708496, 'eval_runtime': 0.1324, 'eval_samples_per_second': 15.108, 'eval_steps_per_second': 7.554, 'epoch': 1.0}\n",
      "{'loss': 2.5119, 'grad_norm': 1.536733865737915, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 1.8856, 'grad_norm': 1.3685024976730347, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 2.8215484619140625, 'eval_runtime': 0.1313, 'eval_samples_per_second': 15.234, 'eval_steps_per_second': 7.617, 'epoch': 2.0}\n",
      "{'loss': 2.0365, 'grad_norm': 1.2905688285827637, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 1.6189, 'grad_norm': 1.1336886882781982, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 2.816497802734375, 'eval_runtime': 0.143, 'eval_samples_per_second': 13.982, 'eval_steps_per_second': 6.991, 'epoch': 3.0}\n",
      "{'train_runtime': 1.9967, 'train_samples_per_second': 3.005, 'train_steps_per_second': 3.005, 'train_loss': 2.2438440124193826, 'epoch': 3.0}\n",
      "fake ai hoax paper made up by gpt-4\n",
      "{'before': [7.704880714416504, 6.162439346313477, 2.8038063049316406, 6.360780715942383, 4.05305290222168, 16.83307456970215], 'after': [6.772946834564209, 5.394711494445801, 2.9839484691619873, 6.627538681030273, 3.9177215099334717, 16.98150634765625]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.8982, 'grad_norm': 1.66070556640625, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 4.0169, 'grad_norm': 1.7133033275604248, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 4.369366645812988, 'eval_runtime': 0.1559, 'eval_samples_per_second': 12.828, 'eval_steps_per_second': 6.414, 'epoch': 1.0}\n",
      "{'loss': 3.5287, 'grad_norm': 1.2428947687149048, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 3.3378, 'grad_norm': 1.4541748762130737, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 4.356468677520752, 'eval_runtime': 0.158, 'eval_samples_per_second': 12.661, 'eval_steps_per_second': 6.33, 'epoch': 2.0}\n",
      "{'loss': 3.0659, 'grad_norm': 0.93393474817276, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 3.0109, 'grad_norm': 1.3038973808288574, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 4.354191780090332, 'eval_runtime': 0.1594, 'eval_samples_per_second': 12.548, 'eval_steps_per_second': 6.274, 'epoch': 3.0}\n",
      "{'train_runtime': 2.2731, 'train_samples_per_second': 2.64, 'train_steps_per_second': 2.64, 'train_loss': 3.476408918698629, 'epoch': 3.0}\n",
      "Hardware Hedging Against Scaling Regime Shifts (self.mlscaling)\n",
      "{'before': [10.767059326171875, 3.284585475921631, 2.225442886352539, 5.182804107666016], 'after': [10.917644500732422, 3.4139397144317627, 2.3915717601776123, 5.278247833251953]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.0168, 'grad_norm': 1.5045863389968872, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 1.9515, 'grad_norm': 1.6995381116867065, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 2.9758172035217285, 'eval_runtime': 0.1776, 'eval_samples_per_second': 11.263, 'eval_steps_per_second': 5.631, 'epoch': 1.0}\n",
      "{'loss': 1.7056, 'grad_norm': 1.089555263519287, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 1.4582, 'grad_norm': 1.0138260126113892, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 2.888021469116211, 'eval_runtime': 0.1766, 'eval_samples_per_second': 11.327, 'eval_steps_per_second': 5.664, 'epoch': 2.0}\n",
      "{'loss': 1.3538, 'grad_norm': 1.3632088899612427, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 1.2299, 'grad_norm': 0.939950168132782, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 2.8587775230407715, 'eval_runtime': 0.178, 'eval_samples_per_second': 11.236, 'eval_steps_per_second': 5.618, 'epoch': 3.0}\n",
      "{'train_runtime': 2.458, 'train_samples_per_second': 2.441, 'train_steps_per_second': 2.441, 'train_loss': 1.619314471880595, 'epoch': 3.0}\n",
      "How to Focus\n",
      "{'before': [11.175151824951172, 3.43853759765625, 8.342260360717773, 8.468477249145508], 'after': [11.017436981201172, 3.291187047958374, 7.970725059509277, 8.81446647644043]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.9858, 'grad_norm': 2.995089054107666, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.6439, 'grad_norm': 1.58451247215271, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.429967164993286, 'eval_runtime': 0.1566, 'eval_samples_per_second': 12.768, 'eval_steps_per_second': 6.384, 'epoch': 1.0}\n",
      "{'loss': 2.4293, 'grad_norm': 1.9177875518798828, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.992, 'grad_norm': 1.236825704574585, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.331143379211426, 'eval_runtime': 0.1591, 'eval_samples_per_second': 12.574, 'eval_steps_per_second': 6.287, 'epoch': 2.0}\n",
      "{'loss': 1.8766, 'grad_norm': 1.3951705694198608, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 2.6741, 'grad_norm': 1.2922844886779785, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.304795742034912, 'eval_runtime': 0.1609, 'eval_samples_per_second': 12.431, 'eval_steps_per_second': 6.216, 'epoch': 3.0}\n",
      "{'train_runtime': 2.2869, 'train_samples_per_second': 2.624, 'train_steps_per_second': 2.624, 'train_loss': 2.7669491370519004, 'epoch': 3.0}\n",
      "Deontic Explorations In \"Paying To Talk To Slaves\"\n",
      "{'before': [7.481788158416748, 3.8988535404205322, 4.159244537353516, 13.049931526184082], 'after': [4.832216262817383, 3.738274574279785, 4.002066612243652, 13.358238220214844]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.8747, 'grad_norm': 2.096508502960205, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.5899, 'grad_norm': 2.015336751937866, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.5862479209899902, 'eval_runtime': 0.1495, 'eval_samples_per_second': 13.375, 'eval_steps_per_second': 6.687, 'epoch': 1.0}\n",
      "{'loss': 2.4316, 'grad_norm': 1.262579321861267, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.7612, 'grad_norm': 1.335434079170227, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.575761556625366, 'eval_runtime': 0.1538, 'eval_samples_per_second': 13.007, 'eval_steps_per_second': 6.503, 'epoch': 2.0}\n",
      "{'loss': 1.8673, 'grad_norm': 1.1828184127807617, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 2.3797, 'grad_norm': 1.230667233467102, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.5691187381744385, 'eval_runtime': 0.1585, 'eval_samples_per_second': 12.622, 'eval_steps_per_second': 6.311, 'epoch': 3.0}\n",
      "{'train_runtime': 2.656, 'train_samples_per_second': 2.259, 'train_steps_per_second': 2.259, 'train_loss': 2.650757590929667, 'epoch': 3.0}\n",
      "By default, capital will matter more than ever after AGI\n",
      "{'before': [12.33033275604248, 10.804149627685547, 5.264718532562256, 5.00636625289917, 2.185818910598755], 'after': [13.265295028686523, 10.46897029876709, 5.712345600128174, 4.91886043548584, 2.082519292831421]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.7673, 'grad_norm': 2.271059036254883, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 2.6765, 'grad_norm': 1.7324353456497192, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.623122215270996, 'eval_runtime': 0.1843, 'eval_samples_per_second': 10.85, 'eval_steps_per_second': 5.425, 'epoch': 1.0}\n",
      "{'loss': 3.4631, 'grad_norm': 2.2982003688812256, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.3217, 'grad_norm': 1.8474977016448975, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.6200013160705566, 'eval_runtime': 0.186, 'eval_samples_per_second': 10.755, 'eval_steps_per_second': 5.378, 'epoch': 2.0}\n",
      "{'loss': 2.9655, 'grad_norm': 1.1774643659591675, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 2.1688, 'grad_norm': 0.7183546423912048, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.624293565750122, 'eval_runtime': 0.1877, 'eval_samples_per_second': 10.657, 'eval_steps_per_second': 5.328, 'epoch': 3.0}\n",
      "{'train_runtime': 2.6891, 'train_samples_per_second': 2.231, 'train_steps_per_second': 2.231, 'train_loss': 2.893831968307495, 'epoch': 3.0}\n",
      "The Plan - 2024 Update\n",
      "{'before': [11.085138320922852, 7.613076686859131, 7.271832466125488, 2.5242795944213867], 'after': [10.461780548095703, 7.329089164733887, 6.638682842254639, 2.3973302841186523]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.4617, 'grad_norm': 2.314183235168457, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.6791, 'grad_norm': 4.621912956237793, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 2.6804332733154297, 'eval_runtime': 0.1894, 'eval_samples_per_second': 10.557, 'eval_steps_per_second': 5.279, 'epoch': 1.0}\n",
      "{'loss': 1.937, 'grad_norm': 1.5637528896331787, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.8931, 'grad_norm': 3.296773672103882, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 2.6785223484039307, 'eval_runtime': 0.1935, 'eval_samples_per_second': 10.335, 'eval_steps_per_second': 5.167, 'epoch': 2.0}\n",
      "{'loss': 1.4123, 'grad_norm': 1.2823187112808228, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 2.5411, 'grad_norm': 1.8647823333740234, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 2.6879079341888428, 'eval_runtime': 0.1877, 'eval_samples_per_second': 10.653, 'eval_steps_per_second': 5.327, 'epoch': 3.0}\n",
      "{'train_runtime': 2.7828, 'train_samples_per_second': 2.156, 'train_steps_per_second': 2.156, 'train_loss': 2.4873830676078796, 'epoch': 3.0}\n",
      "Amazon-backed Anthropic debuts AI agents that can do complex tasks, racing against OpenAI, Microsoft and Google\n",
      "{'before': [7.059969425201416, 7.866158485412598, 4.972960948944092, 1.4543977975845337, 11.503323554992676], 'after': [4.935206890106201, 6.333395004272461, 4.868557929992676, 1.1692737340927124, 11.316805839538574]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.4176, 'grad_norm': 2.2762768268585205, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.2375, 'grad_norm': 2.3001043796539307, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.4287538528442383, 'eval_runtime': 0.1674, 'eval_samples_per_second': 11.949, 'eval_steps_per_second': 5.975, 'epoch': 1.0}\n",
      "{'loss': 2.8886, 'grad_norm': 1.398822546005249, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.4307, 'grad_norm': 1.3295927047729492, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.4019901752471924, 'eval_runtime': 0.169, 'eval_samples_per_second': 11.832, 'eval_steps_per_second': 5.916, 'epoch': 2.0}\n",
      "{'loss': 2.3348, 'grad_norm': 1.320399284362793, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 2.0626, 'grad_norm': 1.3593965768814087, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.4008889198303223, 'eval_runtime': 0.1679, 'eval_samples_per_second': 11.912, 'eval_steps_per_second': 5.956, 'epoch': 3.0}\n",
      "{'train_runtime': 2.8613, 'train_samples_per_second': 2.097, 'train_steps_per_second': 2.097, 'train_loss': 2.7286322514216104, 'epoch': 3.0}\n",
      "OpenAI Email Archives from Musk v. Altman\n",
      "{'before': [9.35188102722168, 3.8830721378326416, 2.591801881790161, 1.859856367111206, 10.639219284057617, 6.248669624328613], 'after': [7.7498250007629395, 3.6932098865509033, 2.136195659637451, 1.9319171905517578, 10.877782821655273, 6.625941276550293]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.5776, 'grad_norm': 4.010377883911133, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.6589, 'grad_norm': 3.6633472442626953, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.2440617084503174, 'eval_runtime': 0.1115, 'eval_samples_per_second': 17.937, 'eval_steps_per_second': 8.968, 'epoch': 1.0}\n",
      "{'loss': 2.9709, 'grad_norm': 2.407945394515991, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.4008, 'grad_norm': 2.5443220138549805, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.1872379779815674, 'eval_runtime': 0.1116, 'eval_samples_per_second': 17.928, 'eval_steps_per_second': 8.964, 'epoch': 2.0}\n",
      "{'loss': 2.2253, 'grad_norm': 1.7101964950561523, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 1.8499, 'grad_norm': 1.7173982858657837, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.1816787719726562, 'eval_runtime': 0.1135, 'eval_samples_per_second': 17.622, 'eval_steps_per_second': 8.811, 'epoch': 3.0}\n",
      "{'train_runtime': 2.6239, 'train_samples_per_second': 2.287, 'train_steps_per_second': 2.287, 'train_loss': 2.7805453737576804, 'epoch': 3.0}\n",
      "President Trump Announces Morgan Ortagus as Deputy Special Presidential Envoy for Middle East Peace\n",
      "{'before': [9.416387557983398, 10.741005897521973, 2.1393628120422363, 13.070490837097168, 1.8174492120742798], 'after': [9.39091968536377, 10.194953918457031, 2.2045726776123047, 13.611124992370605, 1.771489143371582]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.8964, 'grad_norm': 2.3559041023254395, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 2.8855, 'grad_norm': 2.407257556915283, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.4159514904022217, 'eval_runtime': 0.1467, 'eval_samples_per_second': 13.638, 'eval_steps_per_second': 6.819, 'epoch': 1.0}\n",
      "{'loss': 2.2417, 'grad_norm': 1.521421194076538, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.004, 'grad_norm': 1.6388133764266968, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.386564254760742, 'eval_runtime': 0.1444, 'eval_samples_per_second': 13.846, 'eval_steps_per_second': 6.923, 'epoch': 2.0}\n",
      "{'loss': 1.5761, 'grad_norm': 1.3388304710388184, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 1.6097, 'grad_norm': 1.3109463453292847, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.395535469055176, 'eval_runtime': 0.1498, 'eval_samples_per_second': 13.355, 'eval_steps_per_second': 6.677, 'epoch': 3.0}\n",
      "{'train_runtime': 2.7717, 'train_samples_per_second': 2.165, 'train_steps_per_second': 2.165, 'train_loss': 2.202237923940023, 'epoch': 3.0}\n",
      "CDC Report on Missouri H5N1 Serology Testing\n",
      "{'before': [9.302383422851562, 2.164090156555176, 2.530121326446533, 8.904608726501465], 'after': [7.715032577514648, 2.296508312225342, 2.2597248554229736, 10.417621612548828]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 4.6645, 'grad_norm': 2.517502546310425, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.5351, 'grad_norm': 1.9744876623153687, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 4.003051280975342, 'eval_runtime': 0.1857, 'eval_samples_per_second': 10.768, 'eval_steps_per_second': 5.384, 'epoch': 1.0}\n",
      "{'loss': 4.0242, 'grad_norm': 1.5720769166946411, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.7426, 'grad_norm': 1.93308687210083, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.8951995372772217, 'eval_runtime': 0.1857, 'eval_samples_per_second': 10.772, 'eval_steps_per_second': 5.386, 'epoch': 2.0}\n",
      "{'loss': 3.2325, 'grad_norm': 1.8237318992614746, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 2.3779, 'grad_norm': 1.5025017261505127, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.867494821548462, 'eval_runtime': 0.1954, 'eval_samples_per_second': 10.234, 'eval_steps_per_second': 5.117, 'epoch': 3.0}\n",
      "{'train_runtime': 3.1729, 'train_samples_per_second': 1.891, 'train_steps_per_second': 1.891, 'train_loss': 3.429463267326355, 'epoch': 3.0}\n",
      "2024 in AI predictions\n",
      "{'before': [8.183303833007812, 0.7180596590042114, 8.179237365722656, 4.342537879943848, 10.570826530456543], 'after': [5.38284969329834, 0.9092058539390564, 8.8030424118042, 4.543374061584473, 10.412266731262207]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 4.0063, 'grad_norm': 2.618964672088623, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.0173, 'grad_norm': 3.8962526321411133, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 4.496792793273926, 'eval_runtime': 0.1718, 'eval_samples_per_second': 11.638, 'eval_steps_per_second': 5.819, 'epoch': 1.0}\n",
      "{'loss': 3.2503, 'grad_norm': 2.181051731109619, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.2413, 'grad_norm': 1.5369327068328857, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 4.481138229370117, 'eval_runtime': 0.1743, 'eval_samples_per_second': 11.475, 'eval_steps_per_second': 5.737, 'epoch': 2.0}\n",
      "{'loss': 2.4798, 'grad_norm': 1.650524377822876, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 1.8449, 'grad_norm': 1.324534296989441, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 4.495326995849609, 'eval_runtime': 0.1831, 'eval_samples_per_second': 10.923, 'eval_steps_per_second': 5.462, 'epoch': 3.0}\n",
      "{'train_runtime': 3.2972, 'train_samples_per_second': 1.82, 'train_steps_per_second': 1.82, 'train_loss': 2.806656757990519, 'epoch': 3.0}\n",
      "Comment on 'Death and the Gorgon'\n",
      "{'before': [11.654606819152832, 3.051333427429199, 9.083542823791504, 6.945958137512207, 10.134012222290039], 'after': [12.029337882995605, 2.9569177627563477, 9.219869613647461, 7.349301338195801, 10.291199684143066]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.6214, 'grad_norm': 2.26008939743042, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.025, 'grad_norm': 2.695547580718994, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.424791097640991, 'eval_runtime': 0.1737, 'eval_samples_per_second': 11.514, 'eval_steps_per_second': 5.757, 'epoch': 1.0}\n",
      "{'loss': 2.9282, 'grad_norm': 1.7743608951568604, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.0993, 'grad_norm': 1.5769550800323486, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.3480515480041504, 'eval_runtime': 0.1733, 'eval_samples_per_second': 11.538, 'eval_steps_per_second': 5.769, 'epoch': 2.0}\n",
      "{'loss': 2.2028, 'grad_norm': 1.4857354164123535, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 1.6797, 'grad_norm': 1.4504806995391846, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.32250714302063, 'eval_runtime': 0.187, 'eval_samples_per_second': 10.695, 'eval_steps_per_second': 5.348, 'epoch': 3.0}\n",
      "{'train_runtime': 3.2739, 'train_samples_per_second': 1.833, 'train_steps_per_second': 1.833, 'train_loss': 2.5927275021870932, 'epoch': 3.0}\n",
      "debating buying NVDA in 2019\n",
      "{'before': [7.8792829513549805, 5.101133346557617, 0.25373393297195435, 7.965005397796631], 'after': [4.433870792388916, 4.562448501586914, 0.215262308716774, 8.595725059509277]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.4299, 'grad_norm': 2.6792428493499756, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.3305, 'grad_norm': 2.825462818145752, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.3379433155059814, 'eval_runtime': 0.1741, 'eval_samples_per_second': 11.485, 'eval_steps_per_second': 5.742, 'epoch': 1.0}\n",
      "{'loss': 2.7355, 'grad_norm': 1.6066468954086304, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.3417, 'grad_norm': 1.7739102840423584, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.325791597366333, 'eval_runtime': 0.1665, 'eval_samples_per_second': 12.009, 'eval_steps_per_second': 6.005, 'epoch': 2.0}\n",
      "{'loss': 1.9831, 'grad_norm': 1.7341994047164917, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 1.8483, 'grad_norm': 1.8331834077835083, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.3339157104492188, 'eval_runtime': 0.1744, 'eval_samples_per_second': 11.469, 'eval_steps_per_second': 5.735, 'epoch': 3.0}\n",
      "{'train_runtime': 3.3289, 'train_samples_per_second': 1.802, 'train_steps_per_second': 1.802, 'train_loss': 2.611490547657013, 'epoch': 3.0}\n",
      "Human study on AI spear phishing campaigns\n",
      "{'before': [8.617542266845703, 5.865469455718994, 5.855831146240234, 6.869799613952637, 0.7429558634757996, 5.193291664123535], 'after': [4.326836585998535, 5.962702751159668, 6.1263532638549805, 7.080767631530762, 0.37877461314201355, 5.513457298278809]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.0659, 'grad_norm': 2.3972017765045166, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.3097, 'grad_norm': 3.591853618621826, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.3992388248443604, 'eval_runtime': 0.1902, 'eval_samples_per_second': 10.513, 'eval_steps_per_second': 5.256, 'epoch': 1.0}\n",
      "{'loss': 2.3208, 'grad_norm': 1.6523462533950806, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.4739, 'grad_norm': 3.74638032913208, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.375133514404297, 'eval_runtime': 0.1871, 'eval_samples_per_second': 10.691, 'eval_steps_per_second': 5.346, 'epoch': 2.0}\n",
      "{'loss': 1.5714, 'grad_norm': 1.4942113161087036, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 2.0755, 'grad_norm': 2.089600086212158, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.3879997730255127, 'eval_runtime': 0.1873, 'eval_samples_per_second': 10.679, 'eval_steps_per_second': 5.339, 'epoch': 3.0}\n",
      "{'train_runtime': 3.3319, 'train_samples_per_second': 1.801, 'train_steps_per_second': 1.801, 'train_loss': 2.469515860080719, 'epoch': 3.0}\n",
      "My AGI safety research—2024 review, ’25 plans\n",
      "{'before': [10.924260139465332, 4.072440147399902, 2.9389638900756836, 0.10930612683296204, 5.323551177978516], 'after': [10.128201484680176, 2.6714420318603516, 2.3213531970977783, 0.08397671580314636, 5.336966514587402]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.5354, 'grad_norm': 2.6700191497802734, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.1829, 'grad_norm': 3.94592547416687, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 4.003451824188232, 'eval_runtime': 0.1551, 'eval_samples_per_second': 12.895, 'eval_steps_per_second': 6.447, 'epoch': 1.0}\n",
      "{'loss': 2.7582, 'grad_norm': 1.6923402547836304, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.2478, 'grad_norm': 3.04793643951416, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 4.013311862945557, 'eval_runtime': 0.1558, 'eval_samples_per_second': 12.838, 'eval_steps_per_second': 6.419, 'epoch': 2.0}\n",
      "{'loss': 1.8756, 'grad_norm': 1.6792405843734741, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 1.7245, 'grad_norm': 2.353637933731079, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 4.035435199737549, 'eval_runtime': 0.1585, 'eval_samples_per_second': 12.617, 'eval_steps_per_second': 6.309, 'epoch': 3.0}\n",
      "{'train_runtime': 3.2121, 'train_samples_per_second': 1.868, 'train_steps_per_second': 1.868, 'train_loss': 2.5540746450424194, 'epoch': 3.0}\n",
      "Parkinson's Law and the Ideology of Statistics\n",
      "{'before': [11.059673309326172, 0.410934716463089, 4.9378437995910645, 9.082354545593262], 'after': [11.269186019897461, 0.23422664403915405, 5.095401763916016, 9.99195671081543]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.339, 'grad_norm': 2.887451171875, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.7994, 'grad_norm': 3.2563376426696777, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.6911461353302, 'eval_runtime': 0.1747, 'eval_samples_per_second': 11.449, 'eval_steps_per_second': 5.725, 'epoch': 1.0}\n",
      "{'loss': 2.4807, 'grad_norm': 2.2783405780792236, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.7528, 'grad_norm': 3.188812494277954, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.6750943660736084, 'eval_runtime': 0.1684, 'eval_samples_per_second': 11.876, 'eval_steps_per_second': 5.938, 'epoch': 2.0}\n",
      "{'loss': 1.591, 'grad_norm': 2.149153470993042, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 2.279, 'grad_norm': 1.790748953819275, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.6802124977111816, 'eval_runtime': 0.1716, 'eval_samples_per_second': 11.652, 'eval_steps_per_second': 5.826, 'epoch': 3.0}\n",
      "{'train_runtime': 3.6872, 'train_samples_per_second': 1.627, 'train_steps_per_second': 1.627, 'train_loss': 2.7069696386655173, 'epoch': 3.0}\n",
      "Preference Inversion\n",
      "{'before': [11.257996559143066, 2.4081733226776123, 7.490055084228516, 6.412927627563477], 'after': [12.913603782653809, 2.2612903118133545, 7.073019981384277, 8.057035446166992]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.5488, 'grad_norm': 3.1160054206848145, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 4.0606, 'grad_norm': 3.991938591003418, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.7835068702697754, 'eval_runtime': 0.2216, 'eval_samples_per_second': 9.024, 'eval_steps_per_second': 4.512, 'epoch': 1.0}\n",
      "{'loss': 2.808, 'grad_norm': 2.174905300140381, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 3.2191, 'grad_norm': 7.194904804229736, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.7082245349884033, 'eval_runtime': 0.2222, 'eval_samples_per_second': 8.999, 'eval_steps_per_second': 4.5, 'epoch': 2.0}\n",
      "{'loss': 2.043, 'grad_norm': 1.584335207939148, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 2.8653, 'grad_norm': 2.442568778991699, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.646120548248291, 'eval_runtime': 0.2322, 'eval_samples_per_second': 8.613, 'eval_steps_per_second': 4.307, 'epoch': 3.0}\n",
      "{'train_runtime': 4.3763, 'train_samples_per_second': 1.371, 'train_steps_per_second': 1.371, 'train_loss': 3.09080179532369, 'epoch': 3.0}\n",
      "Review: Planecrash\n",
      "{'before': [8.528902053833008, 13.35799789428711, 8.998847961425781, 1.961258888244629, 0.19278351962566376], 'after': [9.244955062866211, 12.997112274169922, 4.955875396728516, 0.008964293636381626, 0.050971198827028275]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.7415, 'grad_norm': 2.612143039703369, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 2.835, 'grad_norm': 3.254910469055176, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.560338258743286, 'eval_runtime': 0.1714, 'eval_samples_per_second': 11.669, 'eval_steps_per_second': 5.835, 'epoch': 1.0}\n",
      "{'loss': 2.8215, 'grad_norm': 2.334237575531006, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 1.6291, 'grad_norm': 1.8375204801559448, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.4723968505859375, 'eval_runtime': 0.171, 'eval_samples_per_second': 11.698, 'eval_steps_per_second': 5.849, 'epoch': 2.0}\n",
      "{'loss': 1.9053, 'grad_norm': 2.1019699573516846, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 1.1471, 'grad_norm': 1.539451241493225, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.4505653381347656, 'eval_runtime': 0.176, 'eval_samples_per_second': 11.361, 'eval_steps_per_second': 5.681, 'epoch': 3.0}\n",
      "{'train_runtime': 3.9229, 'train_samples_per_second': 1.529, 'train_steps_per_second': 1.529, 'train_loss': 2.346584419409434, 'epoch': 3.0}\n",
      "The Field of AI Alignment: A Postmortem, and What To Do About It\n",
      "{'before': [10.715858459472656, 2.8538293838500977, 9.619916915893555, 2.8236703872680664, 1.3537986278533936, 9.357948303222656], 'after': [17.13212776184082, 2.7473714351654053, 9.264034271240234, 3.5286686420440674, 0.48276376724243164, 9.244832992553711]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.3063, 'grad_norm': 3.2994930744171143, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.6343, 'grad_norm': 3.1291298866271973, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.973965644836426, 'eval_runtime': 0.1735, 'eval_samples_per_second': 11.525, 'eval_steps_per_second': 5.763, 'epoch': 1.0}\n",
      "{'loss': 2.4674, 'grad_norm': 2.477537155151367, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.3556, 'grad_norm': 2.0268757343292236, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.917025566101074, 'eval_runtime': 0.1726, 'eval_samples_per_second': 11.589, 'eval_steps_per_second': 5.795, 'epoch': 2.0}\n",
      "{'loss': 1.5868, 'grad_norm': 1.8040292263031006, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 1.7624, 'grad_norm': 1.7323975563049316, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.9284799098968506, 'eval_runtime': 0.1773, 'eval_samples_per_second': 11.28, 'eval_steps_per_second': 5.64, 'epoch': 3.0}\n",
      "{'train_runtime': 3.5944, 'train_samples_per_second': 1.669, 'train_steps_per_second': 1.669, 'train_loss': 2.518795907497406, 'epoch': 3.0}\n",
      "The Intelligence Curse\n",
      "{'before': [11.011114120483398, 0.6101000308990479, 8.701272964477539, 2.934598207473755, 10.553529739379883], 'after': [11.304214477539062, 0.4481358230113983, 7.420818328857422, 2.2388195991516113, 10.681926727294922]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.983, 'grad_norm': 3.2141897678375244, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.3046, 'grad_norm': 3.5254054069519043, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 2.876291275024414, 'eval_runtime': 0.1805, 'eval_samples_per_second': 11.079, 'eval_steps_per_second': 5.54, 'epoch': 1.0}\n",
      "{'loss': 2.1398, 'grad_norm': 2.1053242683410645, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.0996, 'grad_norm': 1.6926547288894653, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 2.890993595123291, 'eval_runtime': 0.1859, 'eval_samples_per_second': 10.761, 'eval_steps_per_second': 5.381, 'epoch': 2.0}\n",
      "{'loss': 1.3094, 'grad_norm': 1.8891417980194092, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 1.5913, 'grad_norm': 1.575945496559143, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 2.9214653968811035, 'eval_runtime': 0.1873, 'eval_samples_per_second': 10.677, 'eval_steps_per_second': 5.338, 'epoch': 3.0}\n",
      "{'train_runtime': 3.8221, 'train_samples_per_second': 1.57, 'train_steps_per_second': 1.57, 'train_loss': 2.2379422187805176, 'epoch': 3.0}\n",
      "The Laws of Large Numbers\n",
      "{'before': [11.152969360351562, 12.381094932556152, 6.883550643920898, 5.000158309936523], 'after': [10.98746109008789, 12.829581260681152, 6.23975944519043, 4.902842998504639]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.4465, 'grad_norm': 3.0251810550689697, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 2.4625, 'grad_norm': 3.3443939685821533, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.2333736419677734, 'eval_runtime': 0.1783, 'eval_samples_per_second': 11.22, 'eval_steps_per_second': 5.61, 'epoch': 1.0}\n",
      "{'loss': 2.6116, 'grad_norm': 2.0492935180664062, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 1.4007, 'grad_norm': 1.8274493217468262, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.223329782485962, 'eval_runtime': 0.1792, 'eval_samples_per_second': 11.162, 'eval_steps_per_second': 5.581, 'epoch': 2.0}\n",
      "{'loss': 1.7269, 'grad_norm': 1.6054573059082031, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 0.9109, 'grad_norm': 1.4826263189315796, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.2280447483062744, 'eval_runtime': 0.1846, 'eval_samples_per_second': 10.834, 'eval_steps_per_second': 5.417, 'epoch': 3.0}\n",
      "{'train_runtime': 3.8591, 'train_samples_per_second': 1.555, 'train_steps_per_second': 1.555, 'train_loss': 2.0931734144687653, 'epoch': 3.0}\n",
      "The subset parity learning problem: much more than you wanted to know\n",
      "{'before': [6.899591445922852, 0.8442123532295227, 0.49031174182891846, 9.700416564941406], 'after': [5.6730499267578125, 0.8620076775550842, 0.47212398052215576, 10.04554271697998]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 4.1168, 'grad_norm': 3.2236294746398926, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.8383, 'grad_norm': 3.7380311489105225, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 2.892657995223999, 'eval_runtime': 0.1945, 'eval_samples_per_second': 10.282, 'eval_steps_per_second': 5.141, 'epoch': 1.0}\n",
      "{'loss': 3.2994, 'grad_norm': 2.064343214035034, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.7109, 'grad_norm': 2.8733603954315186, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 2.893580198287964, 'eval_runtime': 0.1934, 'eval_samples_per_second': 10.341, 'eval_steps_per_second': 5.17, 'epoch': 2.0}\n",
      "{'loss': 2.3681, 'grad_norm': 2.0598092079162598, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 2.1504, 'grad_norm': 2.0436267852783203, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 2.8934695720672607, 'eval_runtime': 0.1953, 'eval_samples_per_second': 10.239, 'eval_steps_per_second': 5.12, 'epoch': 3.0}\n",
      "{'train_runtime': 4.0349, 'train_samples_per_second': 1.487, 'train_steps_per_second': 1.487, 'train_loss': 3.080639600753784, 'epoch': 3.0}\n",
      "What’s the short timeline plan?\n",
      "{'before': [9.168852806091309, 3.9152371883392334, 6.799325942993164, 6.304879188537598], 'after': [4.647651195526123, 4.194304466247559, 7.104080677032471, 6.2407636642456055]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.9721, 'grad_norm': 6.88749885559082, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 2.8245, 'grad_norm': 6.573474407196045, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 2.963982105255127, 'eval_runtime': 0.2374, 'eval_samples_per_second': 8.426, 'eval_steps_per_second': 4.213, 'epoch': 1.0}\n",
      "{'loss': 2.7296, 'grad_norm': 6.738276481628418, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 2.2587, 'grad_norm': 5.604180812835693, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 2.7031450271606445, 'eval_runtime': 0.2389, 'eval_samples_per_second': 8.372, 'eval_steps_per_second': 4.186, 'epoch': 2.0}\n",
      "{'loss': 2.0789, 'grad_norm': 5.735146522521973, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 1.7098, 'grad_norm': 3.3593287467956543, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 2.540477991104126, 'eval_runtime': 0.2464, 'eval_samples_per_second': 8.117, 'eval_steps_per_second': 4.059, 'epoch': 3.0}\n",
      "{'train_runtime': 4.1935, 'train_samples_per_second': 1.431, 'train_steps_per_second': 1.431, 'train_loss': 2.4289310773213706, 'epoch': 3.0}\n",
      "Lorem ipsum\n",
      "{'before': [13.19483470916748, 4.822676658630371, 4.694229602813721, 0.15062260627746582, 0.009142794646322727], 'after': [12.946345329284668, 4.517403602600098, 5.635333061218262, 0.7315716743469238, 0.001257344032637775]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.3232, 'grad_norm': 2.512242555618286, 'learning_rate': 0.001, 'epoch': 0.5}\n",
      "{'loss': 3.0244, 'grad_norm': 2.4470341205596924, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
      "{'eval_loss': 3.506387233734131, 'eval_runtime': 0.1841, 'eval_samples_per_second': 10.862, 'eval_steps_per_second': 5.431, 'epoch': 1.0}\n",
      "{'loss': 2.4132, 'grad_norm': 1.939867615699768, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
      "{'loss': 1.8838, 'grad_norm': 2.473356246948242, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
      "{'eval_loss': 3.5382893085479736, 'eval_runtime': 0.1772, 'eval_samples_per_second': 11.287, 'eval_steps_per_second': 5.644, 'epoch': 2.0}\n",
      "{'loss': 1.4453, 'grad_norm': 1.9782445430755615, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
      "{'loss': 1.3757, 'grad_norm': 1.529227375984192, 'learning_rate': 0.0, 'epoch': 3.0}\n",
      "{'eval_loss': 3.5465643405914307, 'eval_runtime': 0.1858, 'eval_samples_per_second': 10.765, 'eval_steps_per_second': 5.382, 'epoch': 3.0}\n",
      "{'train_runtime': 3.7878, 'train_samples_per_second': 1.584, 'train_steps_per_second': 1.584, 'train_loss': 2.2442721724510193, 'epoch': 3.0}\n",
      "politics is the mind-killer\n",
      "{'before': [11.374530792236328, 3.393611192703247, 13.047080993652344, 3.0926449298858643, 6.4303483963012695, 8.23534870147705], 'after': [11.683854103088379, 3.5927183628082275, 13.514053344726562, 3.821317672729492, 7.558355808258057, 7.392338275909424]}\n"
     ]
    }
   ],
   "source": [
    "data = []\n",
    "for i in tqdm(range(len(df))):\n",
    "    sample = df.iloc[i]\n",
    "    r = learn_sample(sample)\n",
    "    print(sample['title'])\n",
    "    print(dict(before=r['before'], after=r['after']))\n",
    "    data.append(dict(**r, **sample))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtEAAAHmCAYAAAC1XtKPAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAfl9JREFUeJzt3XtYlGX+x/H3DMhBkJMCnjVPSIqKZwtDzdQOtqWZZmq1JmmpvzIzUUvMipQOrllm2bnW1FJbzWq1zM3yiIJiUgKGEoqgKCJnZn5/kLOxeGAUmRn8vK6Ly3iee2a+43fH/fBwP/dtMJvNZkREREREpNKMti5ARERERMTRKESLiIiIiFhJIVpERERExEoK0SIiIiIiVlKIFhERERGxkkK0iIiIiIiVFKJFRERERKykEC0iIiIiYiVnWxdwJTIzz1TbaxmNBvz8PDh58iwmk/ancUTqoWNT/xyfeuj41EPHZ4se+vvXqZbXqW66El1JRqMBg8GA0WiwdSlymdRDx6b+OT710PGph45PPaw6CtEiIiIiIlZSiBYRERERsZJCtIiIiIiIlRSiRURERESspBAtIiIiImIlhWgRERERESspRIuIiIiIWEkhWkRERETESgrRIiIiIiJWUogWEREREbGSQrSIiIiIiJUUokVERERErKQQfQ144YUoXnghytZliIiIiNQYCtEiIiIiIlZSiBYRERERsZKzrQuoLjk5cPDg5f/M4ORkxMsLcnKMlJZW7jGtW5vw8rLudTIyjvHqq/PYtWsHvr5+3HbbYB54YCxDh97BuHETuP32OwEwm80MGXI748dPZODA21i7dg3Lln1MevofeHh40K/fAB5/fCpOTk5WvlMRERERuZRrIkTn5ECXLp6cPm2ogmdzr/RIb28zsbG5lQ7SZrOZmTOn0apVa95//1OysrKIiXkRo9FI37792bx5kyVE79+/j9OnT9O7dzh79sSyYEEMzz47lzZt2pKY+Atz5z5L167dCA/vdzlvUkREREQuQtM57Ehs7E6OHTvKtGkzadq0OZ07d+Wxxx5nxYpl3HzzAHbt2k5e3lkANm36jl69bqB2bQ/c3WszffozhIf3o0GDhvTt25/WrYM4dCjFxu9IREREpGa6Jq5Ee3lBbGzuFU3n2LLvKD/GH6XUZAbAyQhdggK4sX0DPNzP/9do7XSO1NRD5OScZuDAcMsxk8lEYWEhTZo0oW7devz88xb69x/If/6ziQkTJgPQtm0wrq6uvPvuEg4dSiY5OYm0tCN0797zst+viIiIiFzYNRGioSxId+liuuzH9+jRgMnmhny4bj8/7k3HbIaDp7I5vD2JW7o1YVD3JtR2q3VFNZaWltK0aXNeeumVCuc8PDzp1+8Wfvjhexo3bsqpU9nccEMYANu3byUyciqDBt1Gz5438NBDEbzyyktXVIuIiIiIXJimc1ghwK82Dw++nucf7kG3tgEAFBaXsu7n33n6ra2s35ZKYXEl7zo8jyZNmpGRcQwfH18aN25C48ZNOHr0D959dwkGg4H+/Qewc+c2fvjhO2688Sbc3NwAWLt2NbfffifTps3kjjvuolmz5vzxR1qVvGcRERERqUgh+jI0qOvBhLvaM/vBboS0qAvA2YISPv8hmelvbeX73WmUlFp/1bt7957Ur1+f5557huTkJOLj9zB//ou4ubnh5ORE69ZB1KvnzxdfrODmm2+xPM7Ly5uEhHiSk5NISUnmxRfncOJEFkVFRVX2nkVERETkvxSir0Cz+nV44t6OTL+/M60bewNw+mwRn/z7N2a8vY2f9h3F9Occ6spwcnLipZdexWw2ERHxADNnTqNnzxt5/PGpljH9+t2Ck5ORHj1usBz7+98fwdfXj0ceeZAnnngMFxcX7rrrHg4e/LXq3qyIiIiIWBjMZnPlU56dycw8U22v5exsxNfXg+zss5SUVLzKbDabSTh0ki82J3M4I9dyvEHd2gy5qQWd2/hjMFTFEntyuS7VQ7Fv6p/jUw8dn3ro+GzRQ3//OtXyOtXtmrmx8GozGAyEtKhLu+v82P1rJqv+k8Kxk3kcPZHHG6sTaF6/DkPCW9CuuZ/CtIiIiIiDU4iuYkaDga5tAwhtU4+fE47xry2HOJFTyO/HzvDq8niCmvgwNLwlrf6c/iEiIiIijkch+ipxMhrp3aEhPa+vz+a4P1j38+/k5BXz65FTvPhJLB1b1uXum1rQNLBm/opDREREpCaz+sbC1NRUxo4dS2hoKH369GHp0qUXHPvDDz/wt7/9jdDQUAYPHsx3331X7nzXrl0JCgoq93X27Fnr34Udq+VspH/XJswbfwNDw1tQ27Xs55b45BNEvb+Tt75MIONkno2rFBERERFrWBWiTSYTERER+Pr6snr1aubMmcPixYtZu3ZthbGJiYlMnDiRoUOHsmbNGkaMGMH//d//kZiYCEBGRgZnzpxh48aNbNmyxfJVu3btqnlndsbVxYnbezVn3oRe3N6rGS61yv7qdxw4zsx3tvPB14mczCmwcZUiIiIitmPNxdoJEyZUuBi7adOmCuOWLl1Kv379yh374IMPKjx23rx5VtVq1XSOrKwsgoODiYqKwtPTk+bNm9OrVy9iY2MZPHhwubHr1q2jZ8+ejBkzBoBmzZrx/fff8/XXX9O2bVuSk5Px9/enSZMmVhXs6DzcajE0vCX9uzRm3dZUftjzB6UmM/+JT+fnhGP069yI23o1w6u2i61LFREREak25y7WhoSEsHr1alJTU5kyZQqBgYEVciZAcnIyMTEx9OrVy3LM27v8PWdHjhxh0aJF+Pn5lTuelJTEyJEjefTRRy3H3N3drarXqhAdEBDAggULgLIl3Xbv3s3OnTuZPXt2hbF33303xcXFFY6fOVO2LF1SUhLXXXedVcXWJN6ertx/SxsGdm/Cv7b8zk8JRykpNfHvnUfYHJ/OgK5NGNi9KbXdNG1dREREaj5rLtYWFRWRlpZGSEgI/v7+F3zO2bNnExwcTEZGRrnjycnJ3HXXXRd97KVcdkLr168f6enp9O3bl4EDB1Y437Jly3LfHzx4kK1btzJixAigrPj8/HxGjx7NoUOHCA4OZsaMGVYFa2dnI05O1bNfjNFYtixdrVpOVfqajQLqMGFICPf0a8X3sWnsP3QSgF9Ss0k5lsNNHRrSvV0gLs5OVfaa16qr1UOpHuqf41MPHZ966PjsuYfWXKxNSUnBYDBcdEbDmjVryM/P55577uGNN96o8PjmzZtfUb2XHaIXLlxIVlYWUVFRREdHM2vWrAuOPXnyJJMmTaJz587cfPPNQFnxp0+fZsqUKXh6evLOO+/w4IMP8tVXX+Hp6VmpGnx8alf7msuenm5X5Xm9vNxp2dTv0gPlil2tHkr1UP8cn3ro+NRDx2fvPbzUxdqUlBQ8PT2ZNm0aO3bsoH79+kyaNInw8HCgLHu+/PLLvP/+++zbt6/cY7Oysjh16hSrV68mMjISV1dX7rnnHv7+979blSsvO0SHhIQAUFhYyNSpU5k2bRouLhXn8WZlZfHQQw9hNptZuHAhRmPZTz3vvvsuxcXFeHh4APDyyy8THh7Opk2bzjvv5XxOncqr1ivRnp5u5OYWWLWV9+X6/WgOG3elcTjjv7sy+tRxpV/nxnRoWdfyk6RUXnX3UKqW+uf41EPHpx46Plv00MvLurnGcOmLtSkpKRQUFBAWFkZERAQbNmxgwoQJLF++nJCQEF588UXuvvtuWrduXSFEp6SkAFC3bl0WL17MgQMHeP7553FycuLBBx+sdI1W31gYFxdH//79LcdatWpFcXExubm5FSZtZ2RkWG4s/Oijj8qdd3FxKRe6XV1dady4cYU5KxdTUmKqti0rnZ3LwnpxcWm1vGYDv9qMuqU1+1JOsmpzMoePl20lHpt4nIb1PLi7dws6t6mn3Q+tUN09lKql/jk+9dDxqYeOz1F6eKmLtY8++iijR4+23EjYtm1b9u/fz4oVKzh16hRxcXE8//zz533u7t27s23bNnx9fQEICgri5MmTLFu2zKoQbdVl3LS0NCZOnFgu6CYkJODn51chQOfl5fHwww9jNBr55JNPCAwMtJwzm83079+fVatWlRufmppKixYtrCmpRjMYDHRoWZdnH+rGhLvaU9+vbPm/9KyzvLF6H89/tIv9v5/EbNbVABEREXFsWVlZbNy4sdyxv16s/Suj0VhhJY4WLVqQkZHB+vXrOXbsGL169SI0NJTZs2eTnp5OaGgou3btArAE6HNatmxp1YVcsPJKdEhICO3atWPGjBlERkbyxx9/EBMTw/jx4wHIzMykTp06uLm5sWTJEg4fPszHH39sOQfg5uZGnTp16NOnD6+//jqNGjXCz8+Pf/zjH9SvX98yl0X+y2gw0K1tAJ3b1OPnfcf48qdDnMwp5NDRM7zyWRxtm/owJLwlrRppK3ERERFxTOcu1m7evNly8fVCF2unT5+OwWAgOjraciwxMZE2bdowbtw4SzYF+Pe//83HH3/Mxx9/TGBgICtXrmTp0qV88803lt/oHzhwwOoLuVZdiXZycuLNN9/E3d2d4cOHM3PmTEaPHm2ZshEWFsb69esB+PbbbykoKGDYsGGEhYVZvl544QUAnnrqKQYOHMiTTz7JsGHDKCkp4e2338bJSatQXIiT0Ujvjg2JjujFff1b41W7FgCJh0/x4sexLPx8L0eO517iWURERETsz18v1iYlJbF58+YKF2sLCso2puvXrx9r165lzZo1pKamsmjRImJjYxk1ahR169alWbNmlq+6devi7OxMs2bNcHNz44YbbiAzM5N58+aRmprKV199xTvvvMPDDz9sVb0GswPPBcjMPHPpQVXE2dmIr68H2dln7WYOUUFRCRt3pfH19sPkF5YAYAC6Xx/IXWHXEehXM3d/vFz22EOpPPXP8amHjk89dHy26KG/f51Kj83IyGDu3Lls3boVd3d3Ro0axSOPPILBYCAoKIjo6GiGDBkCYLminJ6eTuvWrYmMjKRbt24VnnPVqlUsWrSI77//3nJs165dxMTEkJiYSN26dRk3bhz33XefVe9LIbqS7Pkfjtz8Yr7ZfpiNu45Q9GdtRoOB3h0bMPiG5vh52fcyNtXFnnsol6b+OT710PGph47P3kO0I9F2eDWAp3st7unTklu6Nmbdz6n8EFe2lfjmuHR+2qetxEVERESqmn1tVSNXxNvTlfsHtCE6oic3htTHYMCylfjTb21lzY8p5BWU2LpMEREREYenEF0D1fNxZ+zt1zN3bA+6BJXtCV9YVMq/fvqdp9/6ma+3p1JUXGrjKkVEREQcl0J0DdawngeP3R3CMw90pf11ZUvDnC0oYeWmZKYv2cqmPX9QUqo5bSIiIiLWUoi+BlzXwIspwzvx9MhQy1rSp3KL+PjbX5n5zja2JhzT9q0iIiIiVlCIvoYENfUlclRnHh/WgSYBngBknirgnXW/MPu9Hez+LVO7H4qIiIhUglbnuMaUbSVej/Yt6rIr8Tir/5NCRnY+f2SdZdGqfVzXwIuh4S24vrnfpZ9MRERE5BqlEH2NMhoMdA8OpEuQPz/tO8a/LFuJ5/DyZ3EEN/NlyE0taKmtxEVEREQqUIi+xjkZjdzUsSG92gXyw5501m39nTN5xRxIzeaFj2Pp1KoeQ25qQeM/p3+IiIiIiEK0/KmWsxO3dGtCWIcGbNx1hG92HCa/sJS4pCzik7Lo0S6Qv4VdR6CvthIXERERUYiWctxdnRl843X07dyYr7en8t2uNIpKTGzbn8HOA8fp3aEBg2+8Dt86rrYuVURERMRmFKLlvDzdazGsTytu6dqEdT//zua4dEpNZn6IS2fLvmPc3KURt/VsRh1tJS4iIiLXIC1xJxfl4+nKqAFBvBjRkxva18dA2Vbi3+7471bi+YXaSlxERESuLQrRUin+Pu48fMf1PDe2O13alG0lXmDZSnwr32w/rK3ERURE5JqhEC1WaeTvyWNDyrYSb/fnVuK5+cWs2JTE9CVb+UFbiYuIiMg1QCFaLst1Dbx4cngnpt0XSstGXkDZVuIfndtKfL+2EhcREZGaSyFarkjbZr7MGNWFyfd0oLH/X7YSX/sLs9/fwR5tJS4iIiI1kFbnkCtmMBjo1KoeHVrWZeeB46z+MYXj2fn8kXmW11fto0VDL4be1IJgbSUuIiIiNYRCtFQZo8FAj+vLthL/OeEYX245RPaZQlLSc4g5t5V4eAtaNtRW4iIiIuLYNJ1DqpyzU9lW4i890pMRN7fG070WQNlW4h/F8voXe0nLzLVxlSIiIiKXT1ei5aqp5ezEgG5N6N2hARt2HeHbP7cS33Mwi7iDWfT8cyvxAG0lLiIiIg5GIVquOndXZ+688Tr6dW7M19tS2RibRnGJia37M9hx4Di9OzZk8A3NtZW4iIiIOAyFaKk2nu61GNa3Ff27NmHd1t/5z7mtxPf8wU/7jnJz58bc2rOpthIXERERu6c50VLtfOu4MnpAEC9E9KRXu7KtxItLTHyz4zBPv7WVL7cc0lbiIiIiYtcUosVmAnzcGTe4bCvxzn/ZSvzLLYd4+q2tfLtDW4mLiIiIfVKIFptr5O/JxHNbiTf3Bcq2El/+fRKRb2/jhzhtJS4iIiL2RSFa7MZ1Dbx4ckQoT90XSsuGZVuJZ58p5KNvfmXWO9vZtv8YJu1+KCIiInZAIVrsTnAzX2aM7sLkoR1o7O8BwPFT+by99hei3ttB3MEsbSUuIiIiNqXVOcQuGQwGOrWuR4dWddlxIIM1Px7ieHY+aZlnWfjFXlo29GJIeEuCm/naulQRERG5BilEi10zGgz0vL4+XYMC+GnfUf710+9knykkOT2HmGV7uL65L0NuakmLP6d/iIiIiFQHhWhxCM5ORsI7NaJXu/ps2vMHX21NJTe/mF9+z+aX33cR2roed9/Ugsb+nrYuVURERK4BCtHiUFxqOTGwe1Nu6tiQDTuP8M2OwxQU/XUr8fr8rfd1BPi427pUERERqcEUosUhubs6c2fYdfTr0pj121L5zrKV+DF2HMjgpo4NuUNbiYuIiMhVohAtDs3TvRb39m3FLV2bsPbn3/kxvmwr8U17/mDLvqPc3KUxt/Vshqd7LVuXKiIiIjWIlriTGsG3jitjBgbxwrge9GoX+N+txLcf5um3fuZfP2krcREREak6CtFSowT41mbc4HbMGdud0Nb1AMgvLGXNj4eY+sZPfLfzsI0rFBERkZpAIVpqpMb+nkwa2oGZY7pw/Z9biZ/JK2bBZ3vYvOcPG1cnIiIi55OamsrYsWMJDQ2lT58+LF269IJjJ0yYQFBQULmvTZs2VRi3dOlS+vXrV+5YdnY2kyZNIjQ0lH79+vHll19aXavVIdqaN/fLL78wbNgwOnbsyNChQ0lISCh3ft26dfTv35+OHTvy2GOPcfLkSavfgMjFtGzozdQRoTw1opPlJsP31h9gV+JxG1cmIiIif2UymYiIiMDX15fVq1czZ84cFi9ezNq1a887Pjk5mZiYGLZs2WL5uvHGG8uNOXLkCIsWLarw2MjISM6cOcPy5cuZMGECs2bNYu/evVbVa1WItubN5eXlERERQdeuXVm1ahWhoaE88sgj5OXlAbB3715mzpzJxIkTWb58OTk5OURGRlpVvEhlBTf3Y9rIUOrUdsFshiX/2k/CoRO2LktERET+lJWVRXBwMFFRUTRv3pzw8HB69epFbGxshbFFRUWkpaUREhKCv7+/5cvFxaXcuNmzZxMcHFzu2OHDh9m0aRPPP/88bdq0YdiwYdx5553885//tKpeq1bn+Oub8/T0pHnz5pY3N3jw4HJj169fj6urK9OmTcNgMDBz5kz+85//8M033zBkyBA++eQTbr31Vu666y4A5s+fT9++fTly5AhNmjSpVD1GowGj0WDNW7hsTk7Gcn+K42la34voR2/kjc/jKCo28dW2w9T1cqdJoDZocQT6DDo+9dDxqYeOz557GBAQwIIFCwAwm83s3r2bnTt3Mnv27ApjU1JSMBgMF82Ma9asIT8/n3vuuYc33njDcjw+Pp4GDRrQuHFjy7EuXbqwZMkSq+q1KkRb8+bi4+Pp0qULBkNZyDUYDHTu3Jm4uDiGDBlCfHw848aNs4xv0KABDRs2JD4+vtIh2s/Pw/L81cXLS5t4ODIvL3fmT7rJ1mXIFdBn0PGph45PPXR89t7Dfv36kZ6eTt++fRk4cGCF8ykpKXh6ejJt2jR27NhB/fr1mTRpEuHh4QCcPHmSl19+mffff599+/aVe2xmZiYBAQHljtWtW5eMjAyrarzsH0P69evHyJEjCQ0NPe+bu1CBx44dA+D48eMXPS9yNSUkZ/HEgs08/uoPzFryM5nZebYuSURERP60cOFC3nrrLQ4cOEB0dHSF8ykpKRQUFBAWFsbSpUsJDw9nwoQJlsD84osvcvfdd9O6desKj83Pz68w7cPFxYWioiKrarzszVYWLlxIVlYWUVFRREdHM2vWLKsKLCgouOI3cPLk2WqdzuHl5U5OTj6lpaZqeU2pWn/tYSM/d27u3IglX+4HYNrrPzLrga74ebnZuEq5EH0GHZ966PjUQ8dnix76+npY/ZiQkBAACgsLmTp1KtOmTSuXGx999FFGjx6Nt7c3AG3btmX//v2sWLGCU6dOERcXx/PPP3/e53Z1da2QN4uKinBzsy4DXHaIvtSbu1SBFzrv7l75Xy+YTGZMJvPlvoXLUlpqoqRE/3A4snM97BEcSG5eMZ9u+I2s0wXM+3Q30+/vTJ3aLpd+ErEZfQYdn3ro+NRDx2ePPczKyiIuLo7+/ftbjrVq1Yri4mJyc3Px8/OzHDcajZYAfU6LFi1ISkpi/fr1HDt2jF69egFQUlJCcXExoaGhvPPOOwQGBpKVlVXhtf39/a2q16rpHFlZWWzcuLHcsb++ub+6UIHnpnBU1RsQuRI3d2nMXb2vA+DoiTxeWxGvnQ1FRERsIC0tjYkTJ5abm5yQkICfn1+5AA0wffr0Cqu6JSYm0qJFC6ZOncpXX33FmjVrWLNmDZMnTyYgIIA1a9bQvn17OnXqxB9//FFuCnFsbCydOnWyql6rQrQ1b65jx47s2bMHs7nsSvG5GxE7duxoOf/XJUuOHj3K0aNHLedFqsvgG5ozoFvZzay/HzvD61/spai41MZViYiIXFtCQkJo164dM2bMICkpic2bNxMTE8P48eOBsvvtCgoKgLJ789auXcuaNWtITU1l0aJFxMbGMmrUKOrWrUuzZs0sX3Xr1sXZ2ZlmzZrh5uZGkyZNCAsL46mnniIxMZGVK1eybt067r//fqvqtSpEW/PmBg0aRE5ODi+88AJJSUm88MIL5Ofnc+uttwJw33338eWXX7Jy5UoSExOZNm0affr0qfTKHCJVxWAwMLxfK8JCGgCQePgUb325nxLN9xMREak2Tk5OvPnmm7i7uzN8+HBmzpzJ6NGjGTNmDABhYWGsX78egAEDBjB79mwWL17MHXfcwffff8/SpUvLLVt3MfPnz8fDw4N7772Xt956ixdffJEOHTpYVa/BfO5ScSVlZGQwd+5ctm7diru7O6NGjeKRRx7BYDAQFBREdHQ0Q4YMAco2VJk9ezbJyckEBQUxZ84crr/+estzrVq1ioULF3L69GluvPFG5s6di6+vb6Vrycw8Y03pV8TZ2YivrwfZ2Wftbg6RVM6lelhqMvHWmv3E/pYJQK92gYy943qM1byMopyfPoOOTz10fOqh47NFD/3961TL61Q3q0O0PVGIFmtUpofFJSYWfh7P/t+zAbi5c2NG3tK62tcjl4r0GXR86qHjUw8dn0J01bG/7WpEbKiWs5HHhoTQsqEXAN/tTmPNj4dsXJWIiIjYG4Vokf/h5uLM/w3rSCP/snUt1/78O//ecdjGVYmIiIg9UYgWOQ9P91o8ObwTAT5l65Z/9n0SP8an27gqERERsRcK0SIX4OPpypMjOuHjWbb5ygffJLIr8biNqxIRERF7oBAtchH+Pu48ObwTHm7OmM3w9tr97D900tZliYiIiI0pRItcQiN/T6YM74SrixMlpWZeX7WXpD9O27osERERsSGFaJFKuK6BF5OHdsDZyUhRsYkFK+JJO5576QeKiIhIjaQQLVJJwc18mfC3dhgNBvIKS3hleRwZ2Xm2LktERERsQCFaxAqhbfx56La2AJw+W8Qrn8WRfabQxlWJiIhIdVOIFrHSjSENGNm/NQBZpwt4ZXkcufnFNq5KREREqpNCtMhl6N+1CXeFXQdAetZZXlsRR35hiY2rEhERkeqiEC1ymQbf2JxbujYB4NDRM7z+xV6KS0ptXJWIiIhUB4VokctkMBgYfnMrbgypD0Di4VMsXrOfUpPJxpWJiIjI1aYQLXIFjAYDD97als5t/AGIS8riva8SMZnNNq5MREREriaFaJEr5GQ08sid1xPczBeArfuPsWzjQcwK0iIiIjWWQrRIFajl7MSkoSG0aOgFwHexaXy55ZCNqxIREZGrRSFapIq4uTjz+LCONPL3AOBfP/3Ov3cesXFVIiIicjUoRItUIU/3Wjw5vBP+Pm4AfPbdQX7cm27jqkRERKSqKUSLVDEfT1emjgjF29MFgA++TiT210wbVyUiIiJVSSFa5Crw93Fn6vBOeLg5YzbDkn8lsP/3k7YuS0RERKqIQrTIVdLI35PH7+2Iay0nSkrNLPpiH8l/nLZ1WSIiIlIFFKJFrqKWDb2ZNDQEZycDhcWlLFgZT9rxXFuXJSIiIldIIVrkKru+uR/j/9Yeo8HA2YISXlkex/HsPFuXJSIiIldAIVqkGnRu489Dt7UF4PTZIl7+LI7sM4U2rkpEREQul0K0SDW5MaQB9/VvDUDW6QJeWR5Hbn6xjasSERGRy6EQLVKNbunahL+FXQdAetZZXlsRT35hiY2rEhEREWspRItUsztvbE7/ro0BOHQ0h0Wr9lFcUmrjqkRERMQaCtEi1cxgMDDi5tbc2L4+AAdSs3nry/2Umkw2rkxEREQqSyFaxAaMBgMP3taW0Nb1ANhzMIv31ydiMpttXJmIiIhUhkK0iI04GY2M/1s7gpv5AvBzwjE+23gQs4K0iIiI3VOIFrGhWs5OTBwSwnUNvADYGJvGv3763bZFiYiIyCUpRIvYmLurM0/c25FG9TwA+HLLITbsPGLjqkRERORiFKJF7ICney2mDO9EPW83AJZ9d5Cf9h21cVUiIiJyIQrRInbCt44rU0d0wtvDBYD31yey+7dMG1clIiIi56MQLWJHAnxr8+SITni4OWMym3nrywR++f2krcsSERGR/6EQLWJnGvt78vi9HXGt5URJqZnXv9hHcvppW5clIiJy1aWmpjJ27FhCQ0Pp06cPS5cuveDYCRMmEBQUVO5r06ZNAOTl5TFr1ix69OhBt27deOaZZzh79qzlsRs2bKjw2MmTJ1tVq/PlvUURuZpaNvRm0tAQFqyMp7C4lAUr4nn6/s409ve0dWkiIiJXhclkIiIigpCQEFavXk1qaipTpkwhMDCQwYMHVxifnJxMTEwMvXr1shzz9vYG4MUXXyQhIYF3330Xg8HAjBkzeOmll5g7dy4ASUlJ9O3b1/I9gKurq1X1Wh2iMzIyeOGFF9i2bRuurq7cdtttTJkypcILjx49mh07dlR4/JAhQ4iOjub06dN079693DkfHx+2b99ubUkiNdL1zf145M72vLlmH2cLSnhleRyRo7oQ4ONu69JERESqXFZWFsHBwURFReHp6Unz5s3p1asXsbGxFUJ0UVERaWlphISE4O/vX+G5atWqxTPPPEP79u0BGDp0KJ999pnlfHJyMm3atDnvYyvLqhBtNpuZPHkyXl5efPrpp5w+fZoZM2ZgNBp5+umny419/fXXKS4utnwfHx/P448/zsiRI4GynwB8fHxYt26dZYzRaN3sEqPRgNFosOoxl8vJyVjuT3E8jtjDHu0CcXFx4l9bDgGw8ock/n5bMHVqu9i4surniP2T8tRDx6ceOj577mFAQAALFiwAyjLn7t272blzJ7Nnz64wNiUlBYPBQJMmTc77XH99TFpaGuvWrSt38TY5OZkbbrjhiuq1KkSnpKQQFxfHTz/9RL16ZdsVT548mXnz5lUI0T4+Ppb/Li0t5bXXXuPhhx8mJCTE8lzXXXfdFf0E4OfngcFQPSH6HC8vXQV0dI7Ww37dPejXvZmty7AbjtY/qUg9dHzqoeOz9x7269eP9PR0+vbty8CBAyucT0lJwdPTk2nTprFjxw7q16/PpEmTCA8PLzfu6aefZs2aNTRq1IjHHnsMKAvohw4dYsuWLSxZsoTS0lIGDRrE5MmTcXGp/EUqq0K0v78/S5cutQToc3Jzcy/6uFWrVnH69GnGjRtnOZaUlETz5s2tefkKTp48W61Xor283MnJyae01FQtrylVy9F7uGnPH/wnLh2ARv4ejB4YhGstJxtXVX0cvX+iHtYE6qHjs0UPfX09rH7MwoULycrKIioqiujoaGbNmlXufEpKCgUFBYSFhREREcGGDRuYMGECy5cvt1ywBRg3bhz33Xcfr7zyCuPGjWPVqlUcPXqU/Px8XFxcWLBgAWlpaTz//PMUFBRUeJ2LMZjNZrPV7+xPJpOJkSNH4uvry+LFi887xmw2M2jQIIYOHUpERITl+MMPP0xJSQm5ublkZGTQtWtXIiMjCQgIqPTrnziRqxAtleboPTSbzXzy798suxle39yPKSM64uJ8bQRpR++fqIc1gXro+BwlRJ/zzTffMHXqVHbv3l3uKrHJZOLMmTOWGwkBxo8fj7+/f7mbBc/Jysqid+/efPDBB/To0YNTp07h7e1tmdHw7bff8tRTT7Fnzx6cnCr3/6tXtDpHTEwMv/zyC59//vkFx2zfvp1jx45x7733ljuekpKCn58fkZGRmM1mXnvtNcaPH8/KlSsrXbymc8jlcOQeTrw3lFIzfL/rCL/8fpKl6w4wfUw3u5zbdrU4cv+kjHro+NRDx2ePPczKyiIuLo7+/ftbjrVq1Yri4mJyc3Px8/OzHDcajeUCNECLFi1ISkqiqKiITZs2ceONN+LpWbaqVb169fDx8SE7OxsoP+0YoGXLlhQWFnL69Olyr3Mxlx2iY2Ji+PDDD3nttddo06bNBcd9++233HTTTRWK/eqrrzAYDLi5lW1zvHDhQsLCwoiPj6dz586VqkHTOcQaNaWHowe05lROAbt/y2RbwjFe+WQXYwdfj7Gaf6CsbjWlf9cy9dDxqYeOz56vRKelpTFx4kQ2b95MYGAgAAkJCfj5+VUIttOnT8dgMBAdHW05lpiYSJs2bTAajUyfPp25c+dyxx13AJCenk52djYtW7bkxx9/ZOrUqfzwww+4u5f9MHHgwAF8fHwqHaDhMkP03LlzWbZsGTExMeed7P1XP/74IxMnTqxw/FzR59StWxcfHx8yMjIqXYfJZMZkuuzZKJeltNRESYn+4XBkNaGHj9x5PQtW7uVAajY/7j2Km4szI25uVe2/mbGFmtC/a5166PjUQ8dnjz0MCQmhXbt2zJgxg8jISP744w9iYmIYP348AJmZmdSpUwc3Nzf69evHlClT6NGjB6Ghoaxdu5bY2Fiee+45nJ2dGT58OK+++ir169fHzc2NuXPncvPNN9O6dWsaNGiAq6srs2bN4rHHHuPIkSPMnz+fhx9+2Kp6rf4d8KJFi/jss8949dVXuf322y869uTJkxw5coQuXbqUO56bm0u3bt3Ytm2b5VhGRgbZ2dm0aNHC2pJErjm1nJ2YOCSE6xrUAWDDriOs/el32xYlIiJyBZycnHjzzTdxd3dn+PDhzJw5k9GjRzNmzBgAwsLCWL9+PQADBgxg9uzZLF68mDvuuIPvv/+epUuX0rhxYwCmTJnCwIEDefzxxxkzZgzXXXcd8+bNA8DT05N3332XkydPMnToUGbOnMnw4cOtDtFW3ViYnJzM4MGDiYiI4P777y93zt/fv9xPCFA2H3rcuHHEx8dXuEI2fvx4jh49yty5c3FycuKFF17Aw8ODd955p9LFZ2aeqfTYK+XsbMTX14Ps7LN295ObVE5N7GFufjEvfbqb9KyyrUzv69+aW7qef81MR1cT+3etUQ8dn3ro+GzRQ3//OtXyOtXNqivR3333HaWlpSxevJiwsLByX1D+JwSAEydO4OXldd5fMc+bN4/rr7+eiIgIRo8eTaNGjXj55Zev8O2IXFs83Wvx5PBO1PMu+8F12caD/LTvqI2rEhERqfmuaIk7W9OVaLFGTe7h8ew8oj/ZzemzRRgNBh67uz2hbS5/IyN7VJP7d61QDx2feuj4dCW66lw762KJ1GABvrV5cngnPNycMZnNLP4ygQO/n7R1WSIiIjWWQrRIDdE4wJPHh3XEtZYTJaVmFq7aR0p6jq3LEhERqZEUokVqkJaNvJk4NARnJwOFRaW8tiKOPzJzbV2WiIhIjaMQLVLDtGvuxyN3tsNggLMFJbyyPI7MU/m2LktERKRGUYgWqYG6BAXw4K1tATiVW8TLn+3hVG6hjasSERGpORSiRWqo3h0aMqJfKwAyTxXwyvI4cvOLbVyViIhIzaAQLVKDDejelME3NAfgj8yz/GNlPAVFJbYtSkREpAZQiBap4e7qfR03dynbBjU5PYc3Vu2jWOu7ioiIXBGFaJEazmAwcF//1vRqFwjA/t+zeftf+yk1KUiLiIhcLoVokWuA0WDgoduC6dSqHgCxv2Xy4de/YnLcDUtFRERsSiFa5Brh7GRkwl3taNvUB4At+46y4vskzArSIiIiVlOIFrmG1HJ2YtLQDjSvXweAf+88wtqff7dtUSIiIg5IIVrkGuPu6swT93akYT0PANb8eIjvYtNsXJWIiIhjUYgWuQbVqe3Ck8M7Uc/bDYBPN/zG1oRjNq5KRETEcShEi1yjfOu48uSITnh5uADw7lcH2HMw08ZViYiIOAaFaJFrWKBvbZ4c3onars6YzGYWr9nPgdRsW5clIiJi9xSiRa5xTQI8efzejrjUMlJSamLhF3s5dDTH1mWJiIjYNYVoEaFVI28mDemAk9FAYVEpry6P44+ss7YuS0RExG4pRIsIAO2u8+ORO9thMMDZghJe+WwPWafybV2WiIiIXVKIFhGLrm0DeHBQWwBO5Rbx8mdxnM4ttHFVIiIi9kchWkTK6d2xIcP7tQLg+Kl8Xlkex9mCYhtXJSIiYl8UokWkgoHdm3LHDc0BSMs8y4KV8RQWldq2KBERETuiEC0i53V37+vo17kRAMl/5LBo1V6KS0w2rkpERMQ+KESLyHkZDAZG3tKGnu0CAdj/ezZvr91PqUlBWkRERCFaRC7IaDDw99uC6dSqHgCxv2by4Te/YjabbVyZiIiIbSlEi8hFOTsZGf+3dgQ18QFgy96jLP8+SUFaRESuaQrRInJJLrWcmHxPB5rXrwPAv3ceYd3WVBtXJSIiYjsK0SJSKe6uzjxxb0ca1K0NwOr/pPBdbJqNqxIREbENhWgRqbQ6tV14cngn6nq5AfDpht/Yuv+YjasSERGpfgrRImIVPy83po7ohJeHCwDvrjtA3MEsG1clIiJSvRSiRcRqgX61mXJvR2q7OmMym3lzTQKJqdm2LktERKTaKESLyGVpGliHx4d1xKWWkZJSEwu/2Muhozm2LktERKRaKESLyGVr1dibiXeH4GQ0UFBUymsr4knPOmvrskRERK46hWgRuSLtW9TlkTvbYTBAbn4xryyPI+tUvq3LEhERB5SamsrYsWMJDQ2lT58+LF269IJjJ0yYQFBQULmvTZs2AZCXl8esWbPo0aMH3bp145lnnuHs2f9e5CksLGTGjBl07dqVsLAw3nvvPatrdbb+7YmIlNe1bQAPFLblg68TyT5TyMvL44i8vzPenq62Lk1ERByEyWQiIiKCkJAQVq9eTWpqKlOmTCEwMJDBgwdXGJ+cnExMTAy9evWyHPP29gbgxRdfJCEhgXfffReDwcCMGTN46aWXmDt3LgDz588nISGBDz/8kPT0dJ5++mkaNmzIoEGDKl2vrkSLSJW4qWND7u3bCoDj2fm8sjyeswXFNq5KREQcRVZWFsHBwURFRdG8eXPCw8Pp1asXsbGxFcYWFRWRlpZGSEgI/v7+li8Xl7KVo2rVqsUzzzxD+/btadeuHUOHDrU8T15eHitXrmTmzJm0a9eOW265hYcffphPP/3UqnoVokWkygzq0ZTbezUDIC0zlwUr4yksKrVxVSIi4ggCAgJYsGABnp6emM1mYmNj2blzJ927d68wNiUlBYPBQJMmTc77XLNnz6ZLly4ApKWlsW7dOsvzJCYmUlJSQmhoqGV8ly5diI+Px2QyVbpeq6dzZGRk8MILL7Bt2zZcXV257bbbmDJlCq6uFX9tO2HCBL7//vtyx9566y369u0LwAcffMC7775Lbm4ut956K8888wzu7u6VrsVoNGA0Gqx9C5fFyclY7k9xPOph9bi3Xys8a7uwK/E4AKu2pHBfv9Y4O1/Z37v65/jUQ8enHjo+R+lhv379SE9Pp2/fvgwcOLDC+ZSUFDw9PZk2bRo7duygfv36TJo0ifDw8HLjnn76adasWUOjRo147LHHAMjMzMTX19dy1RqgXr16FBYWcurUKfz8/CpVo1Uh2mw2M3nyZLy8vPj00085ffo0M2bMwGg08vTTT1cYf7G5Kt9++y2LFi0iJiaGunXrEhkZSUxMDM8++2yl6/Hz88BgqJ4QfY6XV+VDvtgn9fDqG3Xb9Yy67fqr8tzqn+NTDx2feuj47L2HCxcuJCsri6ioKKKjo5k1a1a58ykpKRQUFBAWFkZERAQbNmxgwoQJLF++nJCQEMu4cePGcd999/HKK68wbtw4Vq1aRX5+frkADVi+LyoqqnSNBrPZbK7s4OTkZG677TZ++ukn6tWrB8C6deuYN28eP/74Y7mxRUVFdOrUia+++orrrruuwnPdf//99OzZk0mTJgGwa9cuxo4dy7Zt2yp9NfrEidxqvRLt5eVOTk4+paWVv9Qv9kM9rF4lpSZWfJ/EwbTTAIS2rsfgG5tf9g++6p/jUw8dn3ro+GzRQ19fj8t+7DfffMPUqVPZvXt3ueBrMpk4c+aM5eIswPjx4/H397fcPPhXWVlZ9O7dmw8++ICTJ0/y/PPP89NPP1nOn8u427dvx8fHp1K1WXUl2t/fn6VLl1oC9Dm5ubkVxl5srkppaSn79u1j4sSJlmOdOnWiuLiYxMTEcnNULsZkMmMyVfpngCpRWmqipET/cDgy9bD63BV2Ha+uiOe3I6dI/uM0uXnFDOvb8op+g6T+OT710PGph47PHnuYlZVFXFwc/fv3txxr1aoVxcXF5ObmlptmYTQaywVogBYtWpCUlERRURGbNm3ixhtvxNPTEyibruHj40N2djaBgYFkZ2dTUlKCs3NZFM7MzMTNzQ0vL69K12tViPby8qJ3796W700mE5988gk9e/asMPZic1VycnIoLCwkICDgv4U4O+Pj48OxY8cqXY/mRIs11MPq5+xs5MkRnYj+OJbfj53hmx2H8axdizvDKv526lLUP8enHjo+9dDx2XMP09LSmDhxIps3byYwMBCAhIQE/Pz8KsxTnj59OgaDgejoaMuxxMRE2rRpg9FoZPr06cydO5c77rgDgPT0dLKzs2nZsiWNGzfG2dmZuLg4unbtCkBsbCwhISEYjZX/e7midaJjYmL45Zdf+Pzzzyucu9hclXNXss83H8WauSiaEy2XQz2sXr7A8xNuZPobW0g7nsvnPyTjX9eD226wPkiD+lcTqIeOTz10fPbYw5CQENq1a8eMGTOIjIzkjz/+ICYmhvHjxwNlV4vr1KmDm5sb/fr1Y8qUKfTo0YPQ0FDWrl1LbGwszz33HM7OzgwfPpxXX32V+vXr4+bmxty5c7n55ptp3bo1AHfddRdRUVG8+OKLHD9+nPfee69cIK8Mq+ZE/1VMTAzvv/8+r7322nnvmrzYXJUnnniCXr16sX79elq2bGk5f8MNNxAVFcWAAQMqVYPmRIs11EPbOnG6gOc/3MWJnAIMwCN3teeG9vUr/Xj1z/Gph45PPXR89j4nOiMjg7lz57J161bc3d0ZNWoUjzzyCAaDgaCgIKKjoxkyZAgAK1euZOnSpaSnp9O6dWsiIyPp1q0bUHZv3muvvcbatWvJy8tjwIABzJo1yzK9Iz8/n6ioKP7973/j6enJ2LFjefDBB616X5cVoufOncuyZcuIiYnh9ttvr/Tj5s+fT1JSEm+99RadOnXinXfeoUePHgCUlJTQoUMHPv3000rPic7MPGNt6ZfN2dmIr68H2dln7W4OkVSOemh7GSfziP4klpy8YpyMBiYOCaFjq3qXfiDqX02gHjo+9dDx2aKH/v51quV1qpvVE2IWLVrEZ599xquvvnrRAD19+nQiIyPLHUtMTKRFixYYjUZCQkLK7UATFxeHs7Mzbdu2tbYkEXEQgX61mTK8E+6uzpSazLy5JoFfD2fbuiwRERGrWRWik5OTefPNNxk3bhxdunQhMzPT8gVlc1UKCgqAskWy165dy5o1a0hNTWXRokXExsYyatQoAEaOHMm7777Lxo0b2bt3L1FRUdx7771WbbYiIo6naWAdHh/WARdnI8UlJv7x+V5+P5Zj67JERESsYtV0jrfffptXXnnlvOd+/fVXq+aqnHu+Dz74gKKiIgYMGMDs2bPPu/PhhWg6h1hDPbQv+1JOsPDzvZSazHi61yJyVGca1L3wvDn1z/Gph45PPXR8ms5RdS77xkJ7oBAt1lAP7c+OAxks+XI/ZsC3jiuRozpTz/v8v41S/xyfeuj41EPHpxBddexvkUARuWZ0Dw5kzKAgALLPFPLyZ3GcPlv5ZS5FRERsRSFaRGwqvFMjhvUtW+ryeHY+ry6PI6+g2MZViYiIXJxCtIjY3K09mnF7r2YAHDmey4LP91JYXGrjqkRERC5MIVpE7MKQm1rQN7QRAElpp3lj9T5KtJmDiIjYKYVoEbELBoOB+we0ocf1gQAkpJzk7bW/YDI57L3PIiJSgylEi4jdMBoMjL09mA4t6wKwK/E4H32biAMvIiQiIjWUQrSI2BVnJyMT7mpPm8beAPwn/igrf0hWkBYREbuiEC0idse1lhOT7+lIs8CytUW/2X6YdT+n2rgqERGR/1KIFhG7VNvNmSeGd6S+X20AVm5K4uufD9m4KhERkTIK0SJit7xquzB1RCfqerkCsHjVXn7ae9TGVYmIiChEi4id8/Ny48kRodSpXQuzGZb8az/fxabZuiwREbnGKUSLiN2r71ebp+/vjE+dsivSn274jX9tOaSbDUVExGYUokXEITQNrMO8iWHU83YDYM2WQyz77iAmBWkREbEBhWgRcRgN63ky64GuNKznAcDGXWm899UBSk3a2VBERKqXQrSIOBQ/Lzem39+Z6xp4AfBzwjHeWJVAcUmpjSsTEZFriUK0iDgcT/daPHVfJ4Kb+QIQl5TFayviyS8ssXFlIiJyrVCIFhGH5ObizOPDOtKljT8AiYdPMX/ZHnLyimxcmYiIXAsUokXEYdVyNjL+rnaEdWgAQOqxM8z7dDcncwpsXJmIiNR0CtEi4tCcjEYeurUtg7o3BeDoiTxe/CSWoyfO2rgyERGpyRSiRcThGQwGhvVtydDwFgCczCnkpU93k3rsjI0rExGRmkohWkRqBIPBwO29mjNmYBAG4ExeMfP+uZtfD2fbujQREamBFKJFpEbpE9qIR/7WDiejgYKiUl5dEU9cUpatyxIRkRpGIVpEapzuwYFMvqcDLs5GiktMLPpiH1v3H7N1WSIiUoMoRItIjRTSoi5PjuhEbVdnTGYz76z9he9i02xdloiI1BAK0SJSY7Vu7MPT93fGy8MFgE83/Ma/thzCbDbbuDIREXF0CtEiUqM1CfAkclRn6nm7AbBmyyGWbTyISUFaRESugEK0iNR4gb61iRzVhUb1PADYGJvGu+sOUFJqsnFlIiLiqBSiReSa4FvHlafv78x1DbwA2Lr/GG+uTqC4pNTGlYmIiCNSiBaRa4aney2euq8Twc18AYhLyuK1FfHkF5bYuDIREXE0CtEick1xc3Hm8WEd6dLGH4DEw6eYv2wPOXlFNq5MREQciUK0iFxzajkbGX9XO3p3aABA6rEzvPTJbk7mFNi4MhERcRQK0SJyTXIyGnnw1rYM6t4UgGMn83jxk1iOnjhr48pERMQRKESLyDXLYDAwrG9Lhoa3AOBkTiEvfbqb1GNnbFyZiMi1KTU1lbFjxxIaGkqfPn1YunTpBcdOmDCBoKCgcl+bNm0CoKioiHnz5nHTTTfRrVs3HnvsMY4d++/OtRs2bKjw2MmTJ1tVq/PlvUURkZrBYDBwe6/meLjV4uNvf+VMXjHz/rmb/7unA0FNfW1dnojINcNkMhEREUFISAirV68mNTWVKVOmEBgYyODBgyuMT05OJiYmhl69elmOeXt7A7Bw4UI2btzIyy+/jJ+fHzExMUycOJGVK1diMBhISkqib9++zJ071/JYV1dXq+rVlWgREaBPaCMe+Vs7nIwGCopKeXVFPHEHs2xdlojINSMrK4vg4GCioqJo3rw54eHh9OrVi9jY2Apji4qKSEtLIyQkBH9/f8uXi0vZDrWrV6/miSeeoHv37rRq1Yq5c+eyb98+UlNTgbIA3qZNm3KP9fLysqpehWgRkT91Dw5k8j0dcHE2UlxiYtGqfWxNOHbpB4qIyBULCAhgwYIFeHp6YjabiY2NZefOnXTv3r3C2JSUFAwGA02aNKlwzmQyERMTww033FDh3JkzZdP1kpOTad68+RXV69DTOYxGA0ajoVpey8nJWO5PcTzqoWOrrv6FtvHn2Ye6sWzjQQqKSvl+zx+4uDjR4/rAq/q61wJ9Bh2feuj4HKWH/fr1Iz09nb59+zJw4MAK51NSUvD09GTatGns2LGD+vXrM2nSJMLDwzEajRUC9EcffYSvry9BQUGYzWYOHTrEli1bWLJkCaWlpQwaNIjJkydbrmRXhkOHaD8/DwyG6gnR53h5uVfr60nVUw8dW3X0z9fXg45t61/117lW6TPo+NRDx2fvPVy4cCFZWVlERUURHR3NrFmzyp1PSUmhoKCAsLAwIiIi2LBhAxMmTGD58uWEhISUG7tx40bee+895syZg4uLC3/88Qf5+fm4uLiwYMEC0tLSeP755ykoKKjwOhdjMJvNZmveVEZGBi+88ALbtm3D1dWV2267jSlTppx3MvYPP/zAa6+9xuHDh2ncuDGPP/44N998s+V8165dLZfVz9m9ezceHh6VquXEidxqvRLt5eVOTk4+paWmanlNqVrqoWOzRf9O5hTw8be/ciq3bCOW7sEBDOzRFGM1//BeU+gz6PjUQ8dnix76+lYu153PN998w9SpU9m9e3e5q8Qmk4kzZ85YbiQEGD9+PP7+/uVuFty4cSOPP/44I0eOZMaMGZbjp06dwtvb23Ix9ttvv+Wpp55iz549ODk5Vao2q65Em81mJk+ejJeXF59++imnT59mxowZGI1Gnn766XJjExMTmThxItOmTSM8PJwtW7bwf//3f3z++ee0bduWjIwMzpw5w8aNG3Fzc7M8rnbt2pWux2QyYzJZ9TPAFSstNVFSon84HJl66Niqs39etV0Y2b8Nry6P44+ssyT/cZrfj57hodva4mznvwq1Z/oMOj710PHZYw+zsrKIi4ujf//+lmOtWrWiuLiY3Nxc/Pz8LMeNRmO5AA3QokULkpKSLN9/9dVXTJs2jREjRpQL0AA+Pj7lvm/ZsiWFhYWcPn263OtcjFX/L5CSkkJcXBzR0dG0bt2arl27MnnyZNatW1dh7Lp16+jZsydjxoyhWbNm3H///fTo0YOvv/4aKJvQ7e/vT5MmTcrdGVnd0zNERC7Gt44rT9/fmRYNy+7a3rr/GG+uTqCouNTGlYmI1CxpaWlMnDiRjIwMy7GEhAT8/PwqBNvp06cTGRlZ7lhiYiItWpSt+79161amTZvG/fffzzPPPFNu3I8//kiPHj3Iz8+3HDtw4AA+Pj6VDtBgZYj29/dn6dKl1KtXr9zx3NzcCmPvvvtupk6dWuH4uekbSUlJXHfddda8vIiITXi612LqiE5c37xs3ei4pCxeWxFPfmGJjSsTEak5QkJCaNeuHTNmzCApKYnNmzcTExPD+PHjAcjMzKSgoAAou/Fw7dq1rFmzhtTUVBYtWkRsbCyjRo2ipKSEGTNm0K1bN8aNG0dmZqblq6ioiNDQUFxdXZk1axYpKSls3ryZ+fPn8/DDD1tVr9Vzov/KZDIxcuRIfH19Wbx48UXHHjx4kL/97W/84x//4JZbbmH27Nns378fd3d3Dh06RHBwMDNmzLAqWGtOtFhDPXRs9tC/4hITb61JYGficQCa16/D1PtC8fKo/N3c1zJ76KFcGfXQ8dn7nOiMjAzmzp3L1q1bcXd3Z9SoUTzyyCMYDAaCgoKIjo5myJAhAKxcuZKlS5eSnp5O69atiYyMpFu3bsTFxTF8+PDzPv9HH31Ejx49OHjwIC+++CJxcXF4eHgwYsQIHnvsMatmRFxRiJ43bx6ffvopn3/+OW3atLnguJMnTzJy5Ejq1avHRx99hNFoZPTo0Rw7dow5c+bg6enJO++8w969e/nqq6/w9PSs1OubzWZN/xCRalVaauKNz+PZsOMwAI38PZn7yA34+9r3ne4iIlK1LjtEx8TE8P777/Paa6+dd/2+c7KysnjooYcoKipi2bJllrkmRUVFFBcXW1biKCwsJDw8nJkzZ553a8fz0ZVosYZ66NjsqX9ms5nl3yexfmvZzld+XmXzphvUvfw70K8F9tRDuTzqoeOz9yvRjuSy1omeO3cuy5YtIyYm5qIBOiMjgzFjxgBll8//OlnbxcWl3FIlrq6uNG7cuNxk8kvR6hxyOdRDx2Yv/bsnvCW1XZ35/IdkTuYUMveDXTw5vBPN6texdWl2z156KJdPPXR86uGVs3qNpkWLFvHZZ5/x6quvcvvtt19wXF5eHg8//DBGo5FPPvmEwMD/7vZlNpvp378/q1atKjc+NTXVcleliIi9u61nM8YMCsIA5OYXM++fu/n1cLatyxIRkWpg1ZXo5ORk3nzzTSIiIujSpQuZmZmWc/7+/mRmZlKnTh3c3NxYsmQJhw8f5uOPPwawjHVzc6NOnTr06dOH119/nUaNGuHn58c//vEP6tevT3h4eBW+PRGRq6tPp0bUdnXmnbW/UFBUyqsr4pnwt/Z0al3v0g8WERGHZVWI/u677ygtLWXx4sUVVuP49ddfCQsLs9w1+e2331JQUMCwYcPKjbv77rt56aWXeOqpp3B2dubJJ58kNzeXnj178vbbb1d6lxgREXvRPTiQ2q7OLFq9j6JiE4tW7WPs7cH0aq+tw0VEaqorWp3D1jIzz1x6UBVxdjbi6+tBdvZZzSFyUOqhY3OE/iWlnWbBynjy/lw/+r7+rbmlaxMbV2U/HKGHcnHqoeOzRQ/9/WvmvSLat1ZEpIq0auzN0/d3tqwbvWzjQb7ccggHvlYhIiIXoBAtIlKFmgR4MmNUZ+p5uwHw5ZZDLNt4EJOCtIhIjaIQLSJSxQJ8axM5qguN6pWtjboxNo131/1CidbVFRGpMRSiRUSuAt86ZRuwtGzoBcDW/Rm8uTqBouJSG1cmIiJVQSFaROQq8XSvxZMjOtGuuS8AcUlZvLYinvw/bzwUERHHpRAtInIVubk4M/mejnQJ8gfg1yOnmP/PPeTkFdm4MhERuRIK0SIiV1ktZyMT/tae3h0aAJCacYaXPtnNidMFNq5MREQul0K0iEg1MBoNPHhrWwb1aArAsZN5RH8ay9ETZ21cmYiIXA6FaBGRamIwGLi3byvu6dMSgJM5hUR/spvUY9W3cZSIiFQNhWgRkWp2W89mjBkUhAHIzS9m3j938+vhbFuXJSIiVlCIFhGxgT6dGvHI39rhZDRQUFTKK8vj2XMw09ZliYhIJSlEi4jYSPfgQP7vng641DJSUmrijVUJ/Jxw1NZliYhIJShEi4jYUPsWdZk6PJTars6YzGaWrjvAhl1HbF2WiIhcgkK0iIiNtWrszdP3d8bbwwWAZRsPsubHFMxms40rExGRC1GIFhGxA00CPIkc1Zl63m4A/Oun3/nnxoOYFKRFROySQrSIiJ0I8K1N5KguNPL3AOC72DTeXfcLJaUmG1cmIiL/SyFaRMSO+NZx5emRnWnZ0AuArfszeHN1AkXFpTauTERE/kohWkTEzni61+LJEZ1o19wXgLikLF5dEU9eQYmNKxMRkXMUokVE7JCbizOT7+lI1yB/AH47cor5y3aTc7bIxpWJiAgoRIuI2K1azkbG/609N3VsAMDhjFyiP93NidMFNq5MREQUokVE7JjRaOCBQW25tUdTADJO5vHiJ7EcPXHWxpWJiFzbFKJFROycwWBgWN9W3NOnJQDZZwqJ/mQ3vx/LsXFlIiLXLoVoEREHcVvPZjwwKAgDkJtfzPx/7iExNdvWZYmIXJMUokVEHEh4p0aMv6s9TkYDBUWlvLoinj0HM21dlojINUchWkTEwXRrG8D/3dMBl1pGSkpNvLEqgZ8Tjtq6LBGRa4pCtIiIA2rfoi5TR4RS29UZk9nM0nUH2LDziK3LEhG5ZihEi4g4qFaNvJl+f2e8PVwAWPbdQdb8mILZbLZxZSIiNZ9CtIiIA2sc4EnkqM7U83YD4F8//c4/Nx7EpCAtInJVKUSLiDi4AN/aRI7qQiN/DwC+i03j3XW/UFJqsnFlIiI1l0K0iEgN4FvHladHdqZlQy8Atu7P4I1V+ygqLrVxZSIiNZNCtIhIDeHpXoupI0Jp19wXgPjkE7y6Ip68ghIbVyYiUjmpqamMHTuW0NBQ+vTpw9KlSy84dsKECQQFBZX72rRpEwBFRUXMmzePm266iW7duvHYY49x7Ngxy2MLCwuZMWMGXbt2JSwsjPfee8/qWhWiRURqEFcXJybf05GuQf4A/HbkFPOX7SbnbJGNKxMRuTiTyURERAS+vr6sXr2aOXPmsHjxYtauXXve8cnJycTExLBlyxbL14033gjAwoUL2bhxIy+//DLLli2jpKSEiRMnWm68nj9/PgkJCXz44YfMnj2bRYsW8c0331hVr0K0iEgNU8vZyPi/teemjg0AOJyRS/SnuzlxusDGlYmIXFhWVhbBwcFERUXRvHlzwsPD6dWrF7GxsRXGFhUVkZaWRkhICP7+/pYvF5ey1YpWr17NE088Qffu3WnVqhVz585l3759pKamkpeXx8qVK5k5cybt2rXjlltu4eGHH+bTTz+1ql6FaBGRGshoNPDAoLbc2rMpABkn83jxk1iOnjhr48pERM4vICCABQsW4OnpidlsJjY2lp07d9K9e/cKY1NSUjAYDDRp0qTCOZPJRExMDDfccEOFc2fOnCExMZGSkhJCQ0Mtx7t06UJ8fDwmU+VvyHau9Eg7ZDQaMBoN1fJaTk7Gcn+K41EPHZv6d3nu69+GJoF1+G5XGgDLvkti1IA2NKznUe21qIeOTz10fI7Sw379+pGenk7fvn0ZOHBghfMpKSl4enoybdo0duzYQf369Zk0aRLh4eEYjcYKAfqjjz7C19fXMm/a19fXctUaoF69ehQWFnLq1Cn8/PwqVaNDh2g/Pw8MhuoJ0ed4eblX6+tJ1VMPHZv6Z73BN7Vi8E2tbF2GhXro+NRDx2fvPVy4cCFZWVlERUURHR3NrFmzyp1PSUmhoKCAsLAwIiIi2LBhAxMmTGD58uWEhISUG7tx40bee+895syZg4uLC/n5+eUCNGD5vqio8vePOHSIPnnybLVeifbycicnJ59Srb3qkNRDx6b+Xbn9h06y6j8pmExmnJyM3NOnBW2b+lbb66uHjk89dHy26KGvr/W/+ToXhAsLC5k6dSrTpk0rF3wfffRRRo8ejbe3NwBt27Zl//79rFixolyI3rhxI48//jijRo1i2LBhALi6ulYIy+e+d3Nzq3SNVoXojIwMXnjhBbZt24arqyu33XYbU6ZMwdXVtcLYX375hdmzZ/Pbb7/RqlUr5syZQ/v27S3n161bx4IFC8jMzCQsLIy5c+dW+vL5OSaTGZOpenflKi01UVKifzgcmXro2NS/yxfUxIfBNzRj0ap9FBWbeOnj3Tx0W1tuDGlQrXWoh45PPXR89tjDrKws4uLi6N+/v+VYq1atKC4uJjc3t1xONBqNlgB9TosWLUhKSrJ8/9VXXzFt2jRGjBjBjBkzLMcDAwPJzs6mpKQEZ+eyKJyZmYmbmxteXl6VrrfSE2LMZjOTJ08mPz+fTz/9lNdee41NmzaxYMGCCmPz8vKIiIiga9eurFq1itDQUB555BHy8vIA2Lt3LzNnzmTixIksX76cnJwcIiMjK120iIhcnvbX1WXqiFBquzpjMpt596sDbNh5xNZliYiQlpbGxIkTycjIsBxLSEjAz8+vwoXW6dOnV8iOiYmJtGjRAoCtW7cybdo07r//fp555ply44KDg3F2diYuLs5yLDY2lpCQEIzGys8Vr/TIlJQU4uLiiI6OpnXr1nTt2pXJkyezbt26CmPXr1+Pq6sr06ZNo2XLlsycORMPDw/L+nuffPIJt956K3fddRdt27Zl/vz5bN68mSNH9A+5iMjV1qqRN9Pv74y3R9mvRpd9d5A1P6ZY1k8VEbGFkJAQ2rVrx4wZM0hKSmLz5s3ExMQwfvx4oOxqcUFB2VKd/fr1Y+3ataxZs4bU1FQWLVpEbGwso0aNoqSkhBkzZtCtWzfGjRtHZmam5auoqAh3d3fuuusuoqKi2Lt3r2XO9JgxY6yqt9Ih2t/fn6VLl1KvXr1yx3NzcyuMjY+Pp0uXLpab/gwGA507d7Yk/vj4eLp27WoZ36BBAxo2bEh8fLxVxYuIyOVpHOBJ5Ogu+PuUzf/710+/888NBzEpSIuIjTg5OfHmm2/i7u7O8OHDmTlzJqNHj7aE27CwMNavXw/AgAEDmD17NosXL+aOO+7g+++/Z+nSpTRu3JiEhATS09PZunUrYWFh5b727NkDQGRkJO3ateOBBx5gzpw5TJo0iQEDBlhVb6XnRHt5edG7d2/L9yaTiU8++YSePXtWGJuZmUmrVuXvBK9bty4HDx4E4Pjx4wQEBFQ4/9ftGCtDS9yJNdRDx6b+Vb2G9Tx45oFuzP/nbtIyz/Ld7jTyi0p4ePD1OF+Fv2f10PGph47P3nsYGBjIokWLznvu119/Lff9sGHDLDcL/lWnTp0qjP1f7u7uzJs3j3nz5l12rZe9OkdMTAy//PILn3/+eYVzF1o65NydjwUFBRc9X1la4k4uh3ro2NS/quXr68H8yTcxZ+k2fk3N5ueEYxSbzDw9phuutZyuymuqh45PPXR86uGVu6wQHRMTw4cffshrr71GmzZtKpy/0NIh55YNudB5d3frGqol7sQa6qFjU/+urqnDO/GPz+NJSDnJzl8ymPnmFp64txO13apuJVT10PGph47PUZa4cwRW/+s4d+5cli1bRkxMzHl3kIGyS/FZWVnljmVlZVmmcFzovL+/v1W1aIk7uRzqoWNT/64OJ6OBSUM68M66X9iVeJxfD5/ixY93MeXeTnh5uFz6CaygHjo+9dDxqYdXzqoJMYsWLeKzzz7j1Vdf5fbbb7/guI4dO7Jnzx7Lnd5ms5ndu3fTsWNHy/nY2FjL+KNHj3L06FHLeRERqX61nI2Mv7MdN3VsCMDhjFyiP93NidMFNq5MRMT+VDpEJycn8+abbzJu3Di6dOlSbrkQKL/syKBBg8jJyeGFF14gKSmJF154gfz8fG699VYA7rvvPr788ktWrlxJYmIi06ZNo0+fPjRp0uQqvEUREakso9HAA4OCuLVnUwAyTubx4iexHD1x1saViYjYl0qH6O+++47S0lIWL15cYbkQKL/siKenJ0uWLCE2NpYhQ4YQHx/P22+/Te3atQEIDQ3lueee44033uC+++7D29ub6Ojoq/D2RETEWgaDgWF9WjGsT0sAss8UEv3Jbg4dzbFxZSIi9sNgduDV9TMzz1Tbazk7G/H19SA7+6zmEDko9dCxqX+28Z/4dD78JhGzGVxdnJg8tAPBzXwv67nUQ8enHjo+W/TQ379OtbxOdbPPRQJFRMQu3NSxIRP+1h4no4HColJeWxHPnt8ybV2WiIjNKUSLiMhFdW0bwP8N64BLLSMlpSbeWJ3AT/uO2rosERGbUogWEZFLan9dXaaOCMXDzRmT2cy7Xx1gw84jti5LRMRmFKJFRKRSWjXy5un7O+PtWbZu9LLvDrLmxxQc+NYaEZHLphAtIiKV1tjfk8hRXfD3KduB9l8//c4/NxzEpCAtItcYhWgREbFKgI87kaO60Mi/bCvf73ansXTdL5RoG2gRuYYoRIuIiNV8PF2Zfn9nWjbyAmDb/gwWrdpHUXGpjSsTEakeCtEiInJZPNxqMXV4KO2u8wNgb/IJXl0eR15BiY0rExG5+hSiRUTksrm6OPF/93Sga9sAAH5LO838ZbvJOVtk48pERK4uhWgREbkizk5Gxt/Zjps6NgTgcEYu0Z/uJut0vo0rExG5ehSiRUTkihmNBh4YFMRtPZsBkHEyj+hPdpOeddbGlYmIXB0K0SIiUiUMBgP39GnJsD4tAcg+U8hLn+7m0NEcG1cmIlL1FKJFRKRK3dqzGQ/e2haDAXLzi5m/bA8HUrNtXZaISJVSiBYRkSp3U8eGTPhbe5yMBgqLSnltRTyxvx63dVkiIlXG2dYFiIhIzdS1bQDurs4sWrWPwuJSXv98HwYnJzq3qmvr0kRErpiuRIuIyFXT7jo/po7ohIebMyazmQWf7WHlpiRMJm0TLiKOTSFaRESuqpaNvHn6/s74eLoAsPan33lleZzWkhYRh6YQLSIiV11jf0+i/t6d4OZluxseSM0m6v0dHEw7ZdvCREQuk0K0iIhUCz8vN1589EYG9WgKwKncIub/cw//3nEYs1nTO0TEsShEi4hItXF2MjLyljY8eld73FycKDWZ+ez7JBavSSC/sMTW5YmIVJpCtIiIVLuubQN49sFuNPb3AGDXr5k89+Eu0o7n2rgyEZHKUYgWERGbqO9Xm5ljunJD+/pA2Vbhz3+0i58Tjtq4MhGRS1OIFhERm3Gt5cTY24N5YFAQzk5GikpMLF13gI++SaS4pNTW5YmIXJBCtIiI2JTBYCC8UyNmjO5MPW83AH6IS+fFT3aTeSrfxtWJiJyfQrSIiNiF5vW9mP1QNzq2LNvRMPXYGZ77YCfxSVk2rkxEpCKFaBERsRsebrWYdE8Hhoa3wGCAswUl/OPzvXyxOVm7HIqIXVGIFhERu2I0GLi9V3OmjgjFq3YtAL7amsory+M4rV0ORcROKESLiIhdCm7my+yHutOqsTdQtsvhnPd38NuRU7YtTEQEhWgREbFjvnVcmXZfKAO6NQH+u8vht9rlUERsTCFaRETsmrOTkRE3t+axu8t2OTSZzSz/Pok3tcuhSI2TmprK2LFjCQ0NpU+fPixduvSCYydMmEBQUFC5r02bNlUYN2vWLF5//fVyxzZs2FDhsZMnT7aqVmerRouIiNhIl6AAGvt78sbqfaRlniX210zSjufy2N0hNA7wtHV5InKFTCYTERERhISEsHr1alJTU5kyZQqBgYEMHjy4wvjk5GRiYmLo1auX5Zi3t3e5Me+88w4rV65k4sSJ5Y4nJSXRt29f5s6daznm6upqVb26Ei0iIg4j8M9dDm88t8thdj7Pf7SLn/Zpl0MRR5eVlUVwcDBRUVE0b96c8PBwevXqRWxsbIWxRUVFpKWlERISgr+/v+XLxcUFgNzcXCZPnsw777xDgwYNKjw+OTmZNm3alHusl5eXVfUqRIuIiENxreXE3/9nl8N3vzrAh9rlUMShBQQEsGDBAjw9PTGbzcTGxrJz5066d+9eYWxKSgoGg4EmTZqc97nS0tIoLCxk1apV5x2TnJxM8+bNr6heh57OYTQaMBoN1fJaTk7Gcn+K41EPHZv65/iquoc3d23C9c3rsmLTQU7lFpGWeZaP/v0b9/ZthW8d634tK5Wjz6Hjc5Qe9uvXj/T0dPr27cvAgQMrnE9JScHT05Np06axY8cO6tevz6RJkwgPDwegbdu2LFmy5LzPbTabOXToEFu2bGHJkiWUlpYyaNAgJk+ebLmSXRkOHaL9/DwwGKonRJ/j5eVera8nVU89dGzqn+Oryh76+noQ1dq/yp5PKkefQ8dn7z1cuHAhWVlZREVFER0dzaxZs8qdT0lJoaCggLCwMCIiItiwYQMTJkxg+fLlhISEXPS509PTyc/Px8XFhQULFpCWlsbzzz9PQUFBhde5GIPZgdcIOnEit1qvRHt5uZOTk09pqalaXlOqlnro2NQ/x3c1e2gym/lp31G+3/0H/Pn/amEdGtAntCFORvu+4uZI9Dl0fLbooa+vx2U/9ptvvmHq1Kns3r273FVik8nEmTNnyt1IOH78ePz9/cvdLAgwevRounfvzqRJkyzHTp06hbe3t+Vi7LfffstTTz3Fnj17cHJyqlRtDn0l2mQyV/s2sKWlJkpK9A+HI1MPHZv65/iuVg97XV8fHw9XlnyZQE5eMcl/nGb7/mM88rf2eHtU/le0cmn6HDo+e+xhVlYWcXFx9O/f33KsVatWFBcXk5ubi5+fn+W40WissBJHixYtSEpKqtRr+fj4lPu+ZcuWFBYWcvr06XKvczH68VxERGqMc7sctv5zl8PEw6eI0i6HIg4hLS2NiRMnkpGRYTmWkJCAn59fhWA7ffp0IiMjyx1LTEykRYsWl3ydH3/8kR49epCfn285duDAAXx8fCodoOEKQnRRURF33HEH27dvP+/50aNHV1jEOigoyPKGT58+XeFcjx49LrccERERoGyXw6fuC2Vg97I78k//ucvhN9u1y6GIPQsJCaFdu3bMmDGDpKQkNm/eTExMDOPHjwcgMzOTgoICoOzGw7Vr17JmzRpSU1NZtGgRsbGxjBo16pKvExoaiqurK7NmzSIlJYXNmzczf/58Hn74YavqvazpHIWFhTz55JMcPHjwgmNef/11iouLLd/Hx8fz+OOPM3LkSKBskWsfHx/WrVtnGWPUvDUREakCzk5GhvdrTatG3ry3/gD5haWs2JRE8h+neei2YGq7OfRsRpEaycnJiTfffJO5c+cyfPhw3N3dGT16NGPGjAEgLCyM6OhohgwZwoABA5g9ezaLFy8mPT2d1q1bs3TpUho3bnzJ1/H09OTdd9/lxRdfZOjQoXh4eDBixAirQ7TVNxYmJSXx5JNPYjab+fXXX/noo48ueQW5tLSUO++8k1tuuYXHH38cgJUrV/LFF1/w2WefWVXwX2Vmnrnsx1rL2dmIr68H2dln7W4OkVSOeujY1D/HZ6seZpzM443VCaRl5gIQ4OvOY3eH0ES7HFpNn0PHZ4se+vvXqZbXqW5W/yi+Y8cOevTowRNPPEGnTp0q9ZhVq1Zx+vRpxo0bZzmWlJR0xYtca51osYZ66NjUP8dnqx42CvBk9t+78dHXify49yjHs/N54aNdPHBrW3p3bFittTg6fQ4dn3pYda5oibugoKBLXok2m80MGjSIoUOHEhERYTn+8MMPU1JSQm5uLhkZGXTt2pXIyEgCAgIq/fpms7na14kWERHHZDab+ff2wyxZvZfiP6/ADezZjIi7QnCpVbklrUREzrnqk8K2b9/OsWPHuPfee8sdT0lJwc/Pj8jISMxmM6+99hrjx49n5cqVlV6f7+TJs1onWipNPXRs6p/js4cedg+qR8ADXXn9i31knsrn222p/Pr7SSYN7YC/r31vPmEP7KGHcmUcbZ1oe3bVQ/S3337LTTfdVGE9vq+++gqDwYCbmxtQtjNNWFgY8fHxdO7cuVLPrXWi5XKoh45N/XN8tu5hY39Pnn2wK++uO0BcUha/HzvDM0u38/Ad19OpdT2b1eVIbN1DuXLq4ZW76hNifvzxR26++eYKx93d3S0BGqBu3br4+PiUWxtQRETkavBwq8XEoSHc06clBgPkFZaw8Iu9fP5DMqUmBQsRubSrGqJPnjzJkSNH6NKlS7njubm5dOvWjW3btlmOZWRkkJ2dXalFskVERK6U0WDgtp7NeGpEKF5/7mi4flsqr3wWx+mzRTauTkTsXZWG6L8ugg1w8OBBXF1dK6zZ5+npSZcuXYiOjmbv3r3s37+fJ554gt69exMUFFSVJYmIiFxU22a+RD3UjTba5VBErFClITosLIz169dbvj9x4gReXl7nXUFj3rx5XH/99URERDB69GgaNWrEyy+/XJXliIiIVIqPpytT7wtlUPemgHY5FJFLu6Il7mxNm62INdRDx6b+OT5H6WHsr5m8t/4X8gtLAejcxp+/a5dDwHF6KBemzVaqjlbaFhER+YsuQf48+0A3GvuX7Wi4+7dMnvtwJ0eO59q4MhGxJwrRIiIi/yPQrzazxnQhLKQBgGWXw5/2HbVxZSJiLxSiRUREzsOllhN/vz2YB29ti7OTkaISE+9+dYAPvj5AcUmprcsTERtTiBYREbmImzo2ZOboLvj7lO1t8J/4o7zwcSzHT+XbuDIRsSWFaBERkUtoVr8Osx/sRqdWZTsaHs7I5bn3dxJ3MMvGlYmIrShEi4iIVEJtt1pMGhrCMO1yKCIoRIuIiFSawWDg1p7NmHbfeXY5zC20cXUiUp0UokVERKwU1PTPXQ6b+AB/7nL4wU7tcihyDVGIFhERuQw+nq48dV8nBvXQLoci1yKFaBERkcvkZDRyb99WTBwSgrurEyazmRWbkli0ah95BSW2Lk9EriKFaBERkSvUuY0/zz7YjSYBZbsc7jmYxXMf7uRwxhkbVyYiV4tCtIiISBUI9K3NzNH/s8vhx7Fs2atdDkVqIoVoERGRKnJul8OHbm1LLWcjxSUm3ltftsthUbF2ORSpSRSiRUREqljv8+xy+OIn2uVQpCZRiBYREbkKmgaW7XIY2vq/uxzOeX8new5m2rgyEakKCtEiIiJXSW23WkwcUrbLodFgIL+whNe/2MfKH5K0y6GIg1OIFhERuYrO7XL41H2d8P5zl8Ovtx3m5WXa5VDEkSlEi4iIVIOgpr7M/ssuh78eOUXU+zv59XC2bQsTkcuiEC0iIlJNzu1yeOu5XQ7PFhGzLI6vt6dql0MRB6MQLSIiUo2cjEaG9W3FpCEhuLs6YzKbWbkp+c9dDottXZ6IVJJCtIiIiA2EtvFn9oNdy+9y+MEu7XIo4iAUokVERGwk4Nwuhx3+3OXwVNkuhz/uTbdxZSJyKQrRIiIiNuRSy4m/31Z+l8P31yfy/nrtcihizxSiRURE7MC5XQ4DfNwB+HHvUV78OJbj2Xk2rkxEzkchWkRExE40DazDsw92/e8uh8dzmfPBLvb8pl0OReyNQrSIiIgdsexy2Pcvuxyu2sfKTdrlUMSeKESLiIjYGYPBwK09/meXw+3a5VDEnihEi4iI2Kmgpr5EPdSNIO1yKNeI1NRUxo4dS2hoKH369GHp0qUXHDthwgSCgoLKfW3atKnCuFmzZvH666+XO1ZYWMiMGTPo2rUrYWFhvPfee1bX6mz1I0RERKTaeHu6MvW+Tqz6Twpfbzts2eVwaHgLBvVoisFgsHWJIlXCZDIRERFBSEgIq1evJjU1lSlTphAYGMjgwYMrjE9OTiYmJoZevXpZjnl7e5cb884777By5UomTpxY7vj8+fNJSEjgww8/JD09naeffpqGDRsyaNCgSterEC0iImLnnIxGhvVpRatG3ixdd4D8whJW/pBM0h+nGXt7MLXdatm6RJErlpWVRXBwMFFRUXh6etK8eXN69epFbGxshRBdVFREWloaISEh+Pv7V3iu3NxcZsyYwbZt22jQoEG5c3l5eaxcuZJ33nmHdu3a0a5dOw4ePMinn35qVYjWdA4REREHEdq6bJfDptrlUGqggIAAFixYgKenJ2azmdjYWHbu3En37t0rjE1JScFgMNCkSZPzPldaWhqFhYWsWrWqwpjExERKSkoIDQ21HOvSpQvx8fGYrLh516GvRBuNBozG6vk1lpOTsdyf4njUQ8em/jk+9bBqNPT3ZPbfu/P1tlT2HMwCYNl3SdzWsymhbSpekatK6qHjc5Qe9uvXj/T0dPr27cvAgQMrnE9JScHT05Np06axY8cO6tevz6RJkwgPDwegbdu2LFmy5LzPnZmZia+vLy4uLpZj9erVo7CwkFOnTuHn51epGh06RPv5eVT7XDAvL/dqfT2peuqhY1P/HJ96WDUeGNyeB2z02uqh47P3Hi5cuJCsrCyioqKIjo5m1qxZ5c6npKRQUFBAWFgYERERbNiwgQkTJrB8+XJCQkIu+tz5+fnlAjRg+b6oqKjSNTp0iD558my1Xon28nInJyef0lKt0+mI1EPHpv45PvXw6jh6Io+Vm5LIPlO29F19v9oM69sSPy+3Kn8t9dDx2aKHvr4eVj/mXBAuLCxk6tSpTJs2rVzwffTRRxk9erTlRsK2bduyf/9+VqxYcckQ7erqWiEsn/veza3ynxuHDtEmkxmTyVytr1laaqKkRP9wODL10LGpf45PPaxa/t5uPDioLe9+9Qt7DmaR/Mdpdv+Wydjbg+l8laZ3qIeOzx57mJWVRVxcHP3797cca9WqFcXFxeTm5pabZmE0GiusxNGiRQuSkpIu+TqBgYFkZ2dTUlKCs3NZFM7MzMTNzQ0vL69K12vfE2JERETkkmq7OTNxSAj39m1l2eVw0ap9rNAuh+JA0tLSmDhxIhkZGZZjCQkJ+Pn5VZinPH36dCIjI8sdS0xMpEWLFpd8neDgYJydnYmLi7Mci42NJSQkBKOx8tFYIVpERKQGMBgMDOrRtNwuh99sP0zMsjhOaZdDcQAhISG0a9eOGTNmkJSUxObNm4mJiWH8+PFA2dXigoICoOzGw7Vr17JmzRpSU1NZtGgRsbGxjBo16pKv4+7uzl133UVUVBR79+5l48aNvPfee4wZM8aqei87RBcVFXHHHXewffv2C4651E4yH3zwAb179yY0NJQZM2aQn59/ueWIiIgI/93lsG1THwB+0y6H4iCcnJx48803cXd3Z/jw4cycOZPRo0dbwm1YWBjr168HYMCAAcyePZvFixdzxx138P3337N06VIaN25cqdeKjIykXbt2PPDAA8yZM4dJkyYxYMAAq+o1mM1mqycVFxYW8uSTT7JhwwY++ugjevTocd5xAwYMYOLEiRV2knFxceHbb79l5syZxMTEULduXSIjI+nRowfPPvtspevIzKy+dTGdnY34+nqQnX3W7uYQSeWoh45N/XN86mH1KjWZWP2fQ6zflgqAwQBDw1ty6xXscqgeOj5b9NDfv061vE51s/pKdFJSEvfeey+HDx++6Lj/3Unm3Ne5Oys/+ugjHnjgAfr27UuHDh2YM2cOX3zxha5Gi4iIVAEno5F7+rRk0tAQ3F2dMZvh8x+Sef2LfeQVFNu6PBGHZ3WI3rFjBz169GD58uUXHXexnWRKS0vZt28fXbt2tRzr1KkTxcXFJCYmWluSiIiIXEBoa39mP9SNpoFluxzGJWUx54Od2uVQ5ApZvcTdyJEjKzXuYjvJ5OTkUFhYSEBAwH8LcXbGx8eHY8eOVboW7Vgo1lAPHZv65/jUQ9tpWM+DZx/qxiff/sYPe/4g81QBL3wUy5hBQYSHNqr086iHjk89rDpXbZ3oi+0kU69ePYDz7hZjzU4x2rFQLod66NjUP8enHtrOk6O60rFNAIu/iKeoxMS7Xx0g9fhZxg/tgGstp0o/j3ro+NTDK3fVQvTFdpJ54okngIpbKxYVFeHuXvmmasdCsYZ66NjUP8enHtqHLq3r8uxD3Xj9871kZOezcedhfjt8kklDOxDoV/uij1UPHZ+j7FjoCK5aiL7YTjI+Pj64urqSlZVFy5YtASgpKeHUqVP4+1d+dyXtWCiXQz10bOqf41MPba9hXQ+eeaAb760/wO7fMjmckcuz725n7O3XV2qXQ/XQ8amHV+6qTYi52E4yRqORkJAQYmNjLefi4uJwdnambdu2V6skERER+VNtN2ceu7v9X3Y5LC3b5fB77XIoUhlVGqKt2Ulm5MiRvPvuu2zcuJG9e/cSFRXFvffea9V0DhEREbl853Y5nDYyFG/PP3c53HGYmH/u0S6HIpdQpSHamp1kbr/9dh555BGeffZZ/v73v9OhQweeeuqpqixHREREKqFNEx+iHvzLLodpp4l6fyeJqdrlUORCLmvHQnuhHQvFGuqhY1P/HJ96aP9KTSbW/HiIr7aW3+VwUI+mGA0G9bAG0I6FVUeLBIqIiAhQtsvh0PCWTB7agdp/2eVw0Rf7OKtdDkXKUYgWERGRcjq1rsezD3WjWWDZFcS4pCzmvL+T34/l2LgyEfuhEC0iIiIVBPi4M2N0Z27q2BCArNMFzH1/F//8NpGdBzI4dDSHM3lFOPCsUJErctXWiRYRERHHVsvZiQdvbUvrxt58/O2vFJWYWPbvX8uNcallpK6XG3W93ajn7U5dL9ey//Zyp663G96eLhireXdhkeqgEC0iIiIXdWNIA5oG1uHDbxL5/WgOf93nrKjYxNETeRw9kXfexzo7GfCrUxayy8L1n//t5UY9bzd86rji7KRfjIvjUYgWERGRS2oS4EnU37tTx8udlMMnyTiRx4mcAk6cLiDrzz9PnC7gRE4BpX9J2SWlZo6fyuf4qfzzPq/BAL51XC2h+lzA/uuV7VrOTtX1NkUqTSFaREREKs3ZyYi/jzu+nq7nPW8ymzmdW/RnuM7/M1gXknU63xKyi4r/u7Sa2Qwncwo5mVPIwbTT531OLw+XvwTrv4TsP/90d1Wckeqn/9WJiIhIlTEaDPjWccW3jiut8K5w3mw2k5tfTNZfrlyfOF1Q9v2f/51XWFLuMTlni8g5W8Sho+dfHcTDzdkSrMtNGfnzaraHmzMGzcuWKqYQLSIiItXGYDBQp7YLdWq7cF0Dr/OOySsosQTq/4bsfMt/5+SVX7P6bEEJZwtyOXw897zP51rLqdw87PJTRtzw8tDNj2I9hWgRERGxK7XdnKnt5kmTAM/zni8sLuXkeeZjn/vvU2cKMf/P+PSss6RnnT3v8zk7GfDzqhiyz/23bx1XnIy6+VHKU4gWERERh+Jay4kGdT1oUNfjvOdLSk2cPFNY7mbHv87JPplTWPHmx+x8jmef/+bHsikq5+Zlu5ebm13P2w0/3fx4TVKIFhERkRrF2clIgI87AT7u5z1vMpk5lVtYbsrI/87RLir5782PJrOZEzmFnMgphAvc/Ojt4VLhxse/Xs12c1HkqmnUUREREbmmGI1l0zf8vNxo3bjiebPZzJm84vI3Pf41bOfkk19YWu4xp88WcfpsESnpF7n50bJsX8WQrZsfHY9CtIiIiMhfGAwGvDxc8PK42M2PxZYVRc630khu/gVufsy4wM2PLk7lVxX5n6Dt5eGikG1nFKJFRERErFTbrRZN3WrRNLDOec8XFpWWherzhux8TucWlb/5saiUP7LO8scFb340/ndL9QpTRtzxqeOimx+rmUK0iIiISBVzdXGiYT0PGta7yM2P51kj+1zozj7zvzc/msjIzifjojc/nidk//m9Xx03ajkrZFclhWgRERGRaubsZCTAtzYBvrXPe/7czY9Z/7N837mQfTKngOIKNz+Wnf/tSMXnMwBeni4E+Lhz38C2tGpw/ivoUnkK0SIiIiJ25q83P9Kk4nmz2UxOXnGFjWj+GrgLiv5786MZOJ1bxOncIpZv/I2Zo7tU35upoRSiRURERByMwWDA28MFbw8XWjSsePOj2Wwmr7CkwuoiZwuKuS2shQ0qrnkUokVERERqGIPBgIdbLTz+5+ZHZ2cjvr4eZGef/wZGqTzNMBcRERERsZJCtIiIiIiIlRSiRURERESspBAtIiIiImIlhWgRERERESspRIuIiIiIWEkhWkRERETsQmpqKmPHjiU0NJQ+ffqwdOnSC46dMGECQUFB5b42bdpkOf/BBx/Qu3dvQkNDmTFjBvn5/90yfcOGDRUeO3nyZKtq1TrRIiIiImJzJpOJiIgIQkJCWL16NampqUyZMoXAwEAGDx5cYXxycjIxMTH06tXLcszb2xuAb7/9lkWLFhETE0PdunWJjIwkJiaGZ599FoCkpCT69u3L3LlzLY91dXW1ql5diRYRERERm8vKyiI4OJioqCiaN29OeHg4vXr1IjY2tsLYoqIi0tLSCAkJwd/f3/Ll4uICwEcffcQDDzxA37596dChA3PmzOGLL76wXI1OTk6mTZs25R7r5VVx58eLUYgWEREREZsLCAhgwYIFeHp6YjabiY2NZefOnXTv3r3C2JSUFAwGA02aNKlwrrS0lH379tG1a1fLsU6dOlFcXExiYiJQFqKbN29+RfU69HQOo9GA0WioltdycjKW+1Mcj3ro2NQ/x6ceOj710PE5Sg/79etHeno6ffv2ZeDAgRXOp6Sk4OnpybRp09ixYwf169dn0qRJhIeHk5OTQ2FhIQEBAZbxzs7O+Pj4cOzYMcxmM4cOHWLLli0sWbKE0tJSBg0axOTJky1XsivDoUO0n58HBkP1hOhzvLzcq/X1pOqph45N/XN86qHjUw8dn733cOHChWRlZREVFUV0dDSzZs0qdz4lJYWCggLCwsKIiIhgw4YNTJgwgeXLl1OvXj2ACoHYxcWFoqIi0tPTyc/Px8XFhQULFpCWlsbzzz9PQUFBhde5GIcO0SdPnq3WK9FeXu7k5ORTWmqqlteUqqUeOjb1z/Gph45PPXR8tuihr6+H1Y8JCQkBoLCwkKlTpzJt2rRyofjRRx9l9OjRlhsJ27Zty/79+1mxYgVPPPEEUDZv+q+Kiopwd3enUaNGbN++HW9vbwwGA8HBwZhMJp566ikiIyNxcnKqVI0OHaJNJjMmk7laX7O01ERJif7hcGTqoWNT/xyfeuj41EPHZ489zMrKIi4ujv79+1uOtWrViuLiYnJzc/Hz87McNxqNlgB9TosWLUhKSsLHxwdXV1eysrJo2bIlACUlJZw6dQp/f38AfHx8yj22ZcuWFBYWcvr06XKvczH2PSFGRERERK4JaWlpTJw4kYyMDMuxhIQE/Pz8KgTb6dOnExkZWe5YYmIiLVq0wGg0EhISUm5Vj7i4OJydnWnbti0//vgjPXr0KLdu9IEDB/Dx8al0gAaFaBERERGxAyEhIbRr144ZM2aQlJTE5s2biYmJYfz48QBkZmZSUFAAlN14uHbtWtasWUNqaiqLFi0iNjaWUaNGATBy5EjeffddNm7cyN69e4mKiuLee+/F3d2d0NBQXF1dmTVrFikpKWzevJn58+fz8MMPW1WvwWw2V+98iCqUmXmm2l7L2dmIr68H2dln7e7XH1I56qFjU/8cn3ro+NRDx2eLHvr716n02IyMDObOncvWrVtxd3dn1KhRPPLIIxgMBoKCgoiOjmbIkCEArFy5kqVLl5Kenk7r1q2JjIykW7dulud6++23+eCDDygqKmLAgAHMnj3bsqHKwYMHefHFF4mLi8PDw4MRI0bw2GOPWbVghUOHaBERERERW9B0DhERERERKylEi4iIiIhYSSFaRERERMRKCtEiIiIiIlZSiBYRERERsZJCtIiIiIiIlRSiRURERESspBAtIiIiImIlhWgRERERESspRP9FYWEhM2bMoGvXroSFhfHee+9dcOyECRMICgoq97Vp06ZqrFYupqioiDvuuIPt27dfcMwvv/zCsGHD6NixI0OHDiUhIaEaK5SLqUz/9Bm0TxkZGUyePJnu3bvTu3dvoqOjKSwsPO9YfQbtkzU91OfQPqWmpjJ27FhCQ0Pp06cPS5cuveBYfQ4vn7OtC7An8+fPJyEhgQ8//JD09HSefvppGjZsyKBBgyqMTU5OJiYmhl69elmOeXt7V2e5cgGFhYU8+eSTHDx48IJj8vLyiIiIYPDgwbz00kssW7aMRx55hA0bNlC7du1qrFb+V2X6B/oM2iOz2czkyZPx8vLi008/5fTp08yYMQOj0cjTTz9dbqw+g/bJmh6CPof2yGQyERERQUhICKtXryY1NZUpU6YQGBjI4MGDy43V5/AKmcVsNpvNZ8+eNYeEhJi3bdtmOfbGG2+YR40aVWFsYWGhOTg42JySklKdJUolHDx40HznnXeaBw8ebG7Tpk25fv7VypUrzf369TObTCaz2Ww2m0wm8y233GL+4osvqrNc+R+V7Z8+g/YpKSnJ3KZNG3NmZqbl2Nq1a81hYWEVxuozaJ+s6aE+h/YpIyPD/H//93/mM2fOWI499thj5tmzZ1cYq8/hldF0jj8lJiZSUlJCaGio5ViXLl2Ij4/HZDKVG5uSkoLBYKBJkybVXaZcwo4dO+jRowfLly+/6Lj4+Hi6dOmCwWAAwGAw0LlzZ+Li4qqhSrmQyvZPn0H75O/vz9KlS6lXr16547m5uRXG6jNon6zpoT6H9ikgIIAFCxbg6emJ2WwmNjaWnTt30r179wpj9Tm8MprO8afMzEx8fX1xcXGxHKtXrx6FhYWcOnUKPz8/y/GUlBQ8PT2ZNm0aO3bsoH79+kyaNInw8HBblC5/MXLkyEqNy8zMpFWrVuWO1a1b95JTCOTqqmz/9Bm0T15eXvTu3dvyvclk4pNPPqFnz54VxuozaJ+s6aE+h/avX79+pKen07dvXwYOHFjhvD6HV0ZXov+Un59fLkADlu+LiorKHU9JSaGgoICwsDCWLl1KeHg4EyZMYN++fdVWr1yZC/X7f3st9kmfQccQExPDL7/8whNPPFHhnD6DjuFiPdTn0P4tXLiQt956iwMHDhAdHV3hvD6HV0ZXov/k6upa4X805753c3Mrd/zRRx9l9OjRlpsn2rZty/79+1mxYgUhISHVU7BckQv1+397LfZJn0H7FxMTw4cffshrr71GmzZtKpzXZ9D+XaqH+hzav3N9KCwsZOrUqUybNq1caNbn8MroSvSfAgMDyc7OpqSkxHIsMzMTNzc3vLy8yo01Go0V7j5u0aIFGRkZ1VKrXLnAwECysrLKHcvKyiIgIMBGFYk19Bm0b3PnzuX9998nJibmvL9CBn0G7V1leqjPoX3Kyspi48aN5Y61atWK4uLiCnPb9Tm8MgrRfwoODsbZ2bncZPrY2FhCQkIwGsv/NU2fPp3IyMhyxxITE2nRokV1lCpVoGPHjuzZswez2QyULeu0e/duOnbsaOPKpDL0GbRfixYt4rPPPuPVV1/l9ttvv+A4fQbtV2V7qM+hfUpLS2PixInlfphJSEjAz8+v3P1doM/hlVKI/pO7uzt33XUXUVFR7N27l40bN/Lee+8xZswYoOyqdEFBAVA2UX/t2rWsWbOG1NRUFi1aRGxsLKNGjbLlW5BL+GsPBw0aRE5ODi+88AJJSUm88MIL5Ofnc+utt9q4SrkQfQbtX3JyMm+++Sbjxo2jS5cuZGZmWr5An0FHYE0P9Tm0TyEhIbRr144ZM2aQlJTE5s2biYmJYfz48YA+h1XKhsvr2Z28vDzztGnTzJ06dTKHhYWZ33//fcu5Nm3alFs3ccWKFeYBAwaY27dvb7777rvNO3bssEHFcjH/u87w//YwPj7efNddd5lDQkLM99xzj3n//v22KFMu4FL902fQ/ixZssTcpk2b836ZzfoMOgJre6jPoX06duyY+bHHHjN37tzZfOONN5oXL15sWQtan8OqYzCb/7yGLyIiIiIilaLpHCIiIiIiVlKIFhERERGxkkK0iIiIiIiVFKJFRERERKykEC0iIiIiYiWFaBERERERKylEi4iIiIhYSSFaRERERMRKCtEiIg6uX79+vP7667YuQ0TkmqIQLSIiIiJiJYVoERERERErKUSLiFyhM2fO8Mwzz9CzZ0+6dOnCmDFj2LdvHwCvv/469913H2+88QY9evSga9euREZGkpuba3n8qVOnmDNnDuHh4XTo0IERI0awffv2cq/x448/Mnz4cDp27MhNN93Ea6+9RmlpqeV8ZmYmEydOpFOnTvTo0YPo6Ohy50VEpGopRIuIXAGz2cy4ceM4cuQIS5YsYcWKFXTq1In77ruPX375BYB9+/axZcsW3nvvPd544w127tzJ448/DkBpaSl///vf2bVrFzExMaxatYo2bdowduxY9u7dC8CePXuIiIigS5curFq1iueff57PPvuMN99801LH559/Trdu3Vi7di1PPfUUH3zwAatXr672vw8RkWuFwWw2m21dhIiIo9q6dSsPPfQQ27Ztw8fHx3J81KhRNG7cmEaNGvH222+zceNGAgMDAfjPf/7DuHHj+Prrrzly5AgRERGsXbuWNm3aAGXBfMiQITRt2pR//OMfTJkyhfT0dD777DPL83/77becOHGCkSNH0q9fPzp06MCCBQss5++88046d+5MVFRUdfw1iIhcc5xtXYCIiCPbv38/ZrOZvn37ljteVFREYWEhjRo1onnz5pYADdC5c2cAfvvtN44cOUKdOnUsARrAYDDQtWtXtmzZYhl34403lnv+gQMHlvu+efPm5b739vamsLDwit+fiIicn0K0iMgVMJlMeHp6smrVqgrnXFxc+Pzzz6lVq1a54+fmKjs5OXGhXwaazWacncv+iT7358U4OTmd9zlEROTq0JxoEZEr0KZNG3JzcykuLqZZs2aWr3feeYfvvvsOgEOHDnHmzBnLY/bs2QPA9ddfT1BQEGfOnOG3336znDebzcTGxtKqVSsAWrZsablR8ZwPP/yQYcOGXe23JyIiF6AQLSJyBXr37k1wcDBPPPEE27ZtIzU1lejoaFatWkXLli0ByMvLY9q0afz222/8/PPPPPfcc9x22200atSIsLAwgoODefLJJ9mxYwfJyck899xz/PbbbzzwwAMAPPzww8TFxfGPf/yD33//nc2bN/Pmm2/Sp08fG75zEZFrm6ZziIhcAScnJ9577z1iYmJ4/PHHyc/Pp2XLlixatIhevXqxa9cuGjRoQHBwMPfffz9OTk4MHjyYqVOnlnv8vHnzmDhxIkVFRbRv354PPviATp06ARAcHMwbb7zBwoULeeeddwgICGDMmDFMmDDBhu9cROTaptU5RESuotdff53Vq1fz/fff27oUERGpQprOISIiIiJiJYVoEREREREraTqHiIiIiIiVdCVaRERERMRKCtEiIiIiIlZSiBYRERERsZJCtIiIiIiIlRSiRURERESspBAtIiIiImIlhWgRERERESspRIuIiIiIWOn/AamsonF49z8DAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 800x550 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# example training\n",
    "df_hist = data[-1]['hist']#.groupby('epoch').last().dropna(axis=1).drop(columns=['step'])\n",
    "df_hist['loss'].plot(label='train')\n",
    "plt.twinx()\n",
    "df_hist['eval_loss'].plot(c='b', label='eval')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_hist['learning_rate'].plot(logy=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Perplexity\n",
    "\n",
    "Perplexity measures how well a language model predicts a text sample. Lower is better\n",
    "\n",
    "It’s calculated as the average number of bits per word a model needs to represent the same\n",
    "\n",
    "https://huggingface.co/docs/transformers/perplexity\n",
    "https://thegradient.pub/understanding-evaluation-metrics-for-language-models/\n",
    "\n",
    "The **improvement** column, is perplexity decrease"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_res = pd.DataFrame(data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sum\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "train/diff_sum      -0.474407\n",
       "train/diff_mean     -0.383682\n",
       "train/diff%_sum     -0.253703\n",
       "train/diff%_mean    -0.253703\n",
       "train/diff_std      -0.226926\n",
       "train/diff%_std     -0.184180\n",
       "train/diff_min      -0.082520\n",
       "train/after_std      0.016230\n",
       "train/before_max     0.026255\n",
       "train/diff%_max      0.056223\n",
       "train/diff_max       0.059453\n",
       "train/after_mean     0.073286\n",
       "train/after_max      0.082784\n",
       "train/before_min     0.126469\n",
       "train/after_min      0.176288\n",
       "train/after_sum      0.182927\n",
       "train/before_std     0.281097\n",
       "train/diff%_min      0.342018\n",
       "train/before_mean    0.388784\n",
       "train/before_sum     0.455190\n",
       "novelty              1.000000\n",
       "Name: novelty, dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_res = pd.DataFrame(data)\n",
    "for stat in ['mean', 'std', 'min', 'max', 'sum']:\n",
    "    agg = getattr(np, stat)\n",
    "    df_res[f'train/before_{stat}'] = df_res['train/before'].apply(lambda x: agg(x))\n",
    "    df_res[f'train/after_{stat}'] = df_res['train/after'].apply(lambda x: agg(x)) \n",
    "\n",
    "    # df_res[f'before_{stat}'] = df_res['before'].apply(lambda x: agg(x))\n",
    "    # df_res[f'after_{stat}'] = df_res['after'].apply(lambda x: agg(x))\n",
    "\n",
    "    # df_res[f\"diff_{stat}\"] = - df_res[f'before_{stat}'] + df_res[f'after_{stat}']\n",
    "    # df_res[f\"diff%_{stat}\"] = df_res[f\"diff_{stat}\"] / df_res[f'before_{stat}'] * 100\n",
    "\n",
    "    df_res[f\"train/diff_{stat}\"] = -df_res[f'train/before_{stat}'] + df_res[f'train/after_{stat}']\n",
    "    df_res[f\"train/diff%_{stat}\"] = df_res[f\"train/diff_{stat}\"] / df_res[f'train/before_{stat}'] * 100\n",
    "\n",
    "r = df_res.select_dtypes(include=np.number).corr()['novelty'].sort_values()\n",
    "print(stat)\n",
    "display(r)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "only lesswrong\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1883910/1966928096.py:2: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n",
      "  m = df_res.url.str.contains('lesswrong').fillna(False)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "train/diff_sum      -0.247629\n",
       "train/diff_mean     -0.159304\n",
       "train/before_max    -0.077736\n",
       "train/diff_std      -0.041644\n",
       "train/diff%_sum     -0.034815\n",
       "train/diff%_mean    -0.034815\n",
       "train/before_min    -0.007299\n",
       "train/after_min     -0.005681\n",
       "train/diff%_std     -0.001165\n",
       "train/diff_min       0.007247\n",
       "train/after_std      0.111126\n",
       "train/after_mean     0.168236\n",
       "train/after_max      0.178731\n",
       "train/before_std     0.202097\n",
       "train/after_sum      0.261667\n",
       "train/diff_max       0.264107\n",
       "train/diff%_max      0.274232\n",
       "train/diff%_min      0.318505\n",
       "train/before_mean    0.390433\n",
       "train/before_sum     0.418176\n",
       "novelty              1.000000\n",
       "Name: novelty, dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "without lesswrong\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "train/diff_sum      -0.524070\n",
       "train/diff_mean     -0.289291\n",
       "train/diff%_sum     -0.212006\n",
       "train/diff%_mean    -0.212006\n",
       "train/diff_std      -0.205211\n",
       "train/diff%_max     -0.173305\n",
       "train/diff_max      -0.153685\n",
       "train/diff%_std     -0.135741\n",
       "train/diff_min       0.065099\n",
       "train/after_sum      0.075106\n",
       "train/after_mean     0.077967\n",
       "train/before_min     0.133288\n",
       "train/after_std      0.148123\n",
       "train/before_mean    0.209628\n",
       "train/before_sum     0.217732\n",
       "train/after_max      0.219580\n",
       "train/before_max     0.319630\n",
       "train/before_std     0.337727\n",
       "train/after_min      0.384317\n",
       "train/diff%_min      0.484895\n",
       "novelty              1.000000\n",
       "Name: novelty, dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "only new\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "train/diff_sum      -0.445037\n",
       "train/diff_mean     -0.296686\n",
       "train/diff_std      -0.175885\n",
       "train/diff%_sum     -0.169818\n",
       "train/diff%_mean    -0.169818\n",
       "train/diff%_std     -0.149809\n",
       "train/diff_min      -0.063278\n",
       "train/after_std     -0.029953\n",
       "train/before_max    -0.012773\n",
       "train/after_mean     0.102790\n",
       "train/after_max      0.116678\n",
       "train/before_min     0.129897\n",
       "train/diff%_max      0.131804\n",
       "train/diff_max       0.133860\n",
       "train/before_std     0.172009\n",
       "train/after_min      0.238759\n",
       "train/after_sum      0.306655\n",
       "train/before_mean    0.357417\n",
       "train/diff%_min      0.367850\n",
       "train/before_sum     0.536616\n",
       "novelty              1.000000\n",
       "Name: novelty, dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# also try with only new stuff, no  less wrong, only less wrong...\n",
    "m = df_res.url.str.contains('lesswrong').fillna(False)\n",
    "r = df_res[m].select_dtypes(include=np.number).corr()['novelty'].sort_values()\n",
    "print('only lesswrong')\n",
    "display(r)\n",
    "\n",
    "r = df_res[~m].select_dtypes(include=np.number).corr()['novelty'].sort_values()\n",
    "print('without lesswrong')\n",
    "display(r)\n",
    "\n",
    "r = df_res[~df_res.in_training].select_dtypes(include=np.number).corr()['novelty'].sort_values()\n",
    "print('only new')\n",
    "display(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f</th>\n",
       "      <th>train/diff_sum</th>\n",
       "      <th>novelty</th>\n",
       "      <th>in_training</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>../samples/2025_lw_review-planecrash.md</td>\n",
       "      <td>-339.481621</td>\n",
       "      <td>0.933950</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>../samples/2025_lw_2024-in-ai-predictions.md</td>\n",
       "      <td>-297.344904</td>\n",
       "      <td>0.789190</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>../samples/2025_lw_the-field-of-ai-alignment-a...</td>\n",
       "      <td>-288.831029</td>\n",
       "      <td>0.925483</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>../samples/2025_lw_the-intelligence-curse.md</td>\n",
       "      <td>-265.925183</td>\n",
       "      <td>0.688044</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>../samples/2025_lw_my-agi-safety-research-2024...</td>\n",
       "      <td>-261.054373</td>\n",
       "      <td>0.756925</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>../samples/2025_lw_comment-on-death-and-the-go...</td>\n",
       "      <td>-239.929921</td>\n",
       "      <td>0.750253</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>../samples/2025_lw_preference-inversion.md</td>\n",
       "      <td>-238.890490</td>\n",
       "      <td>0.633321</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>../samples/2025_lw_debating-buying-nvda-in-201...</td>\n",
       "      <td>-229.822011</td>\n",
       "      <td>0.526975</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>../samples/2025_lw_the-subset-parity-learning-...</td>\n",
       "      <td>-224.939272</td>\n",
       "      <td>0.697946</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>../samples/2024_lesswrong_slop.md</td>\n",
       "      <td>-201.960423</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>../samples/2025_lw_human-study-on-ai-spear-phi...</td>\n",
       "      <td>-200.020488</td>\n",
       "      <td>0.677458</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>../samples/2024_openai_emails.md</td>\n",
       "      <td>-194.183420</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>../samples/2024_news_anthropic.md</td>\n",
       "      <td>-176.985800</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>../samples/2024_lw_by-default-capital-will-mat...</td>\n",
       "      <td>-169.866388</td>\n",
       "      <td>0.906388</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>../samples/2024_deliberative_alignment.md</td>\n",
       "      <td>-159.913571</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>../samples/2025_h5n1_report.md</td>\n",
       "      <td>-156.453212</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>../samples/2024_anthropic_palintir.md</td>\n",
       "      <td>-143.663636</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>../samples/2024_gwern_reddit.md</td>\n",
       "      <td>-142.434464</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>../samples/2024_lw_the-plan-2024-update.md</td>\n",
       "      <td>-136.823458</td>\n",
       "      <td>0.785170</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>../samples/2024_how_to_focus.md</td>\n",
       "      <td>-135.387340</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>../samples/2024_trump_appointment.md</td>\n",
       "      <td>-128.444646</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>../samples/lorem_ipsum.md</td>\n",
       "      <td>-127.469837</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>../samples/2024_bob_fanfic2.md</td>\n",
       "      <td>-125.559481</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>../samples/2024_gpt4_fake_paper.md</td>\n",
       "      <td>-122.651137</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>../samples/politics_is_the_mind_killer.md</td>\n",
       "      <td>-116.203043</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>../samples/2025_lw_the-laws-of-large-numbers.md</td>\n",
       "      <td>-107.898850</td>\n",
       "      <td>0.540932</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>../samples/2024_bob_fanfic.md</td>\n",
       "      <td>-106.220317</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>../samples/2025_lw_what-s-the-short-timeline-p...</td>\n",
       "      <td>-102.163854</td>\n",
       "      <td>0.898161</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>../samples/2025_lw_parkinson-s-law-and-the-ide...</td>\n",
       "      <td>-97.869906</td>\n",
       "      <td>0.677458</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>../samples/2024_arxiv_meh.md</td>\n",
       "      <td>-73.838489</td>\n",
       "      <td>0.150000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    f  train/diff_sum  \\\n",
       "22            ../samples/2025_lw_review-planecrash.md     -339.481621   \n",
       "15       ../samples/2025_lw_2024-in-ai-predictions.md     -297.344904   \n",
       "23  ../samples/2025_lw_the-field-of-ai-alignment-a...     -288.831029   \n",
       "24       ../samples/2025_lw_the-intelligence-curse.md     -265.925183   \n",
       "19  ../samples/2025_lw_my-agi-safety-research-2024...     -261.054373   \n",
       "16  ../samples/2025_lw_comment-on-death-and-the-go...     -239.929921   \n",
       "21         ../samples/2025_lw_preference-inversion.md     -238.890490   \n",
       "17  ../samples/2025_lw_debating-buying-nvda-in-201...     -229.822011   \n",
       "26  ../samples/2025_lw_the-subset-parity-learning-...     -224.939272   \n",
       "8                   ../samples/2024_lesswrong_slop.md     -201.960423   \n",
       "18  ../samples/2025_lw_human-study-on-ai-spear-phi...     -200.020488   \n",
       "12                   ../samples/2024_openai_emails.md     -194.183420   \n",
       "11                  ../samples/2024_news_anthropic.md     -176.985800   \n",
       "9   ../samples/2024_lw_by-default-capital-will-mat...     -169.866388   \n",
       "4           ../samples/2024_deliberative_alignment.md     -159.913571   \n",
       "14                     ../samples/2025_h5n1_report.md     -156.453212   \n",
       "0               ../samples/2024_anthropic_palintir.md     -143.663636   \n",
       "6                     ../samples/2024_gwern_reddit.md     -142.434464   \n",
       "10         ../samples/2024_lw_the-plan-2024-update.md     -136.823458   \n",
       "7                     ../samples/2024_how_to_focus.md     -135.387340   \n",
       "13               ../samples/2024_trump_appointment.md     -128.444646   \n",
       "28                          ../samples/lorem_ipsum.md     -127.469837   \n",
       "3                      ../samples/2024_bob_fanfic2.md     -125.559481   \n",
       "5                  ../samples/2024_gpt4_fake_paper.md     -122.651137   \n",
       "29          ../samples/politics_is_the_mind_killer.md     -116.203043   \n",
       "25    ../samples/2025_lw_the-laws-of-large-numbers.md     -107.898850   \n",
       "2                       ../samples/2024_bob_fanfic.md     -106.220317   \n",
       "27  ../samples/2025_lw_what-s-the-short-timeline-p...     -102.163854   \n",
       "20  ../samples/2025_lw_parkinson-s-law-and-the-ide...      -97.869906   \n",
       "1                        ../samples/2024_arxiv_meh.md      -73.838489   \n",
       "\n",
       "     novelty  in_training  \n",
       "22  0.933950        False  \n",
       "15  0.789190        False  \n",
       "23  0.925483        False  \n",
       "24  0.688044        False  \n",
       "19  0.756925        False  \n",
       "16  0.750253        False  \n",
       "21  0.633321        False  \n",
       "17  0.526975        False  \n",
       "26  0.697946        False  \n",
       "8   0.100000        False  \n",
       "18  0.677458        False  \n",
       "12  0.700000        False  \n",
       "11  0.500000        False  \n",
       "9   0.906388        False  \n",
       "4   0.600000        False  \n",
       "14  0.750000        False  \n",
       "0   0.200000        False  \n",
       "6   1.000000        False  \n",
       "10  0.785170        False  \n",
       "7   0.500000        False  \n",
       "13  0.300000        False  \n",
       "28  0.000000         True  \n",
       "3   0.400000         True  \n",
       "5   0.000000        False  \n",
       "29  0.500000         True  \n",
       "25  0.540932        False  \n",
       "2   0.300000        False  \n",
       "27  0.898161        False  \n",
       "20  0.677458        False  \n",
       "1   0.150000        False  "
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main_metric = 'train/diff_sum'\n",
    "df_res[['f', main_metric, 'novelty', 'in_training']].sort_values( main_metric)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>novelty</th>\n",
       "      <th>date</th>\n",
       "      <th>in_training</th>\n",
       "      <th>train/before_mean</th>\n",
       "      <th>train/after_mean</th>\n",
       "      <th>train/diff_mean</th>\n",
       "      <th>train/diff%_mean</th>\n",
       "      <th>train/before_std</th>\n",
       "      <th>train/after_std</th>\n",
       "      <th>...</th>\n",
       "      <th>train/diff_min</th>\n",
       "      <th>train/diff%_min</th>\n",
       "      <th>train/before_max</th>\n",
       "      <th>train/after_max</th>\n",
       "      <th>train/diff_max</th>\n",
       "      <th>train/diff%_max</th>\n",
       "      <th>train/before_sum</th>\n",
       "      <th>train/after_sum</th>\n",
       "      <th>train/diff_sum</th>\n",
       "      <th>train/diff%_sum</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>../samples/2024_trump_appointment.md</th>\n",
       "      <td>President Trump Announces Morgan Ortagus as De...</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>2025-01-04 00:00:00+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.602360</td>\n",
       "      <td>1.976479</td>\n",
       "      <td>-1.625882</td>\n",
       "      <td>-45.133784</td>\n",
       "      <td>3.130189</td>\n",
       "      <td>2.116151</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.005089</td>\n",
       "      <td>-82.696441</td>\n",
       "      <td>17.917255</td>\n",
       "      <td>11.689822</td>\n",
       "      <td>-6.227433</td>\n",
       "      <td>-34.756625</td>\n",
       "      <td>284.586479</td>\n",
       "      <td>156.141833</td>\n",
       "      <td>-128.444646</td>\n",
       "      <td>-45.133784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2024_how_to_focus.md</th>\n",
       "      <td>How to Focus</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>2024-06-01 00:00:00+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2.030182</td>\n",
       "      <td>1.238443</td>\n",
       "      <td>-0.791739</td>\n",
       "      <td>-38.998412</td>\n",
       "      <td>2.826893</td>\n",
       "      <td>2.380078</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000099</td>\n",
       "      <td>-40.660140</td>\n",
       "      <td>13.456800</td>\n",
       "      <td>13.169463</td>\n",
       "      <td>-0.287336</td>\n",
       "      <td>-2.135250</td>\n",
       "      <td>347.161163</td>\n",
       "      <td>211.773822</td>\n",
       "      <td>-135.387340</td>\n",
       "      <td>-38.998412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2024_gpt4_fake_paper.md</th>\n",
       "      <td>fake ai hoax paper made up by gpt-4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2024-01-01 00:00:00+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2.890543</td>\n",
       "      <td>1.876897</td>\n",
       "      <td>-1.013646</td>\n",
       "      <td>-35.067661</td>\n",
       "      <td>2.491279</td>\n",
       "      <td>2.017982</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000804</td>\n",
       "      <td>-71.474606</td>\n",
       "      <td>11.427372</td>\n",
       "      <td>9.812000</td>\n",
       "      <td>-1.615372</td>\n",
       "      <td>-14.135986</td>\n",
       "      <td>349.755680</td>\n",
       "      <td>227.104543</td>\n",
       "      <td>-122.651137</td>\n",
       "      <td>-35.067661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2025_h5n1_report.md</th>\n",
       "      <td>CDC Report on Missouri H5N1 Serology Testing</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>2025-01-05 00:00:00+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2.840106</td>\n",
       "      <td>1.588481</td>\n",
       "      <td>-1.251626</td>\n",
       "      <td>-44.069677</td>\n",
       "      <td>2.737036</td>\n",
       "      <td>2.004289</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.002226</td>\n",
       "      <td>-77.666175</td>\n",
       "      <td>11.928956</td>\n",
       "      <td>9.404486</td>\n",
       "      <td>-2.524470</td>\n",
       "      <td>-21.162542</td>\n",
       "      <td>355.013296</td>\n",
       "      <td>198.560084</td>\n",
       "      <td>-156.453212</td>\n",
       "      <td>-44.069677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2024_arxiv_meh.md</th>\n",
       "      <td>TradingAgents: Multi-Agents LLM Financial Trad...</td>\n",
       "      <td>0.150000</td>\n",
       "      <td>2024-12-28 00:00:00+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.329921</td>\n",
       "      <td>2.693382</td>\n",
       "      <td>-0.636539</td>\n",
       "      <td>-19.115730</td>\n",
       "      <td>3.075030</td>\n",
       "      <td>2.940876</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000330</td>\n",
       "      <td>-61.776143</td>\n",
       "      <td>13.523319</td>\n",
       "      <td>13.253697</td>\n",
       "      <td>-0.269622</td>\n",
       "      <td>-1.993755</td>\n",
       "      <td>386.270819</td>\n",
       "      <td>312.432330</td>\n",
       "      <td>-73.838489</td>\n",
       "      <td>-19.115730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2024_lw_by-default-capital-will-matter-more-than-ever-after-agi.md</th>\n",
       "      <td>By default, capital will matter more than ever...</td>\n",
       "      <td>0.906388</td>\n",
       "      <td>2024-12-30 19:47:19.838000+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2.935460</td>\n",
       "      <td>1.667801</td>\n",
       "      <td>-1.267660</td>\n",
       "      <td>-43.184356</td>\n",
       "      <td>2.497172</td>\n",
       "      <td>2.004676</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.007733</td>\n",
       "      <td>-65.756404</td>\n",
       "      <td>12.706497</td>\n",
       "      <td>11.517975</td>\n",
       "      <td>-1.188522</td>\n",
       "      <td>-9.353658</td>\n",
       "      <td>393.351672</td>\n",
       "      <td>223.485284</td>\n",
       "      <td>-169.866388</td>\n",
       "      <td>-43.184356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2024_news_anthropic.md</th>\n",
       "      <td>Amazon-backed Anthropic debuts AI agents that ...</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>2025-01-05 05:03:00+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2.437276</td>\n",
       "      <td>1.420116</td>\n",
       "      <td>-1.017160</td>\n",
       "      <td>-41.733471</td>\n",
       "      <td>2.880203</td>\n",
       "      <td>2.245643</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000029</td>\n",
       "      <td>-71.181132</td>\n",
       "      <td>12.790111</td>\n",
       "      <td>12.667040</td>\n",
       "      <td>-0.123071</td>\n",
       "      <td>-0.962233</td>\n",
       "      <td>424.085983</td>\n",
       "      <td>247.100184</td>\n",
       "      <td>-176.985800</td>\n",
       "      <td>-41.733471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2025_lw_the-laws-of-large-numbers.md</th>\n",
       "      <td>The Laws of Large Numbers</td>\n",
       "      <td>0.540932</td>\n",
       "      <td>2025-01-04 18:06:02.387000+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2.887975</td>\n",
       "      <td>2.153969</td>\n",
       "      <td>-0.734006</td>\n",
       "      <td>-25.415931</td>\n",
       "      <td>2.576897</td>\n",
       "      <td>2.052082</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.001162</td>\n",
       "      <td>-29.038092</td>\n",
       "      <td>11.118690</td>\n",
       "      <td>8.321499</td>\n",
       "      <td>-2.797191</td>\n",
       "      <td>-25.157557</td>\n",
       "      <td>424.532354</td>\n",
       "      <td>316.633504</td>\n",
       "      <td>-107.898850</td>\n",
       "      <td>-25.415931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/politics_is_the_mind_killer.md</th>\n",
       "      <td>politics is the mind-killer</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>2007-02-19 00:00:00+00:00</td>\n",
       "      <td>True</td>\n",
       "      <td>3.285058</td>\n",
       "      <td>2.430624</td>\n",
       "      <td>-0.854434</td>\n",
       "      <td>-26.009715</td>\n",
       "      <td>2.930715</td>\n",
       "      <td>2.513867</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.004398</td>\n",
       "      <td>-65.298689</td>\n",
       "      <td>11.889731</td>\n",
       "      <td>11.973468</td>\n",
       "      <td>0.083736</td>\n",
       "      <td>0.704275</td>\n",
       "      <td>446.767846</td>\n",
       "      <td>330.564804</td>\n",
       "      <td>-116.203043</td>\n",
       "      <td>-26.009715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2024_lesswrong_slop.md</th>\n",
       "      <td>Deontic Explorations In \"Paying To Talk To Sla...</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>2024-04-12 00:00:00+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.078750</td>\n",
       "      <td>1.695459</td>\n",
       "      <td>-1.383291</td>\n",
       "      <td>-44.930269</td>\n",
       "      <td>2.946400</td>\n",
       "      <td>2.349493</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000778</td>\n",
       "      <td>-75.973899</td>\n",
       "      <td>15.304575</td>\n",
       "      <td>12.805290</td>\n",
       "      <td>-2.499285</td>\n",
       "      <td>-16.330311</td>\n",
       "      <td>449.497469</td>\n",
       "      <td>247.537046</td>\n",
       "      <td>-201.960423</td>\n",
       "      <td>-44.930269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2025_lw_parkinson-s-law-and-the-ideology-of-statistics-1.md</th>\n",
       "      <td>Parkinson's Law and the Ideology of Statistics</td>\n",
       "      <td>0.677458</td>\n",
       "      <td>2025-01-04 22:59:57.376000+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.550214</td>\n",
       "      <td>2.779585</td>\n",
       "      <td>-0.770629</td>\n",
       "      <td>-21.706555</td>\n",
       "      <td>3.111395</td>\n",
       "      <td>2.862761</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.004117</td>\n",
       "      <td>-90.026555</td>\n",
       "      <td>12.598433</td>\n",
       "      <td>12.334391</td>\n",
       "      <td>-0.264043</td>\n",
       "      <td>-2.095839</td>\n",
       "      <td>450.877187</td>\n",
       "      <td>353.007281</td>\n",
       "      <td>-97.869906</td>\n",
       "      <td>-21.706555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2024_anthropic_palintir.md</th>\n",
       "      <td>Anthropic and Palantir Partner to Bring Claude...</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>2024-07-11 00:00:00+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.575668</td>\n",
       "      <td>2.444458</td>\n",
       "      <td>-1.131210</td>\n",
       "      <td>-31.636323</td>\n",
       "      <td>3.246451</td>\n",
       "      <td>2.673633</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000074</td>\n",
       "      <td>-75.181568</td>\n",
       "      <td>13.271792</td>\n",
       "      <td>12.891874</td>\n",
       "      <td>-0.379918</td>\n",
       "      <td>-2.862598</td>\n",
       "      <td>454.109781</td>\n",
       "      <td>310.446145</td>\n",
       "      <td>-143.663636</td>\n",
       "      <td>-31.636323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2025_lw_preference-inversion.md</th>\n",
       "      <td>Preference Inversion</td>\n",
       "      <td>0.633321</td>\n",
       "      <td>2025-01-03 23:49:06.168000+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.358938</td>\n",
       "      <td>1.602391</td>\n",
       "      <td>-1.756548</td>\n",
       "      <td>-52.294728</td>\n",
       "      <td>2.745416</td>\n",
       "      <td>1.724408</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.003750</td>\n",
       "      <td>-88.162939</td>\n",
       "      <td>11.404502</td>\n",
       "      <td>8.079728</td>\n",
       "      <td>-3.324774</td>\n",
       "      <td>-29.153170</td>\n",
       "      <td>456.815627</td>\n",
       "      <td>217.925137</td>\n",
       "      <td>-238.890490</td>\n",
       "      <td>-52.294728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2024_deliberative_alignment.md</th>\n",
       "      <td>Deliberative Alignment: Reasoning Enables Safe...</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>2024-12-20 00:00:00+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.623786</td>\n",
       "      <td>2.384146</td>\n",
       "      <td>-1.239640</td>\n",
       "      <td>-34.208421</td>\n",
       "      <td>3.441515</td>\n",
       "      <td>2.720267</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000561</td>\n",
       "      <td>-29.432378</td>\n",
       "      <td>14.758335</td>\n",
       "      <td>12.259495</td>\n",
       "      <td>-2.498840</td>\n",
       "      <td>-16.931722</td>\n",
       "      <td>467.468432</td>\n",
       "      <td>307.554861</td>\n",
       "      <td>-159.913571</td>\n",
       "      <td>-34.208421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2025_lw_the-subset-parity-learning-problem-much-more-than-you-wanted.md</th>\n",
       "      <td>The subset parity learning problem: much more ...</td>\n",
       "      <td>0.697946</td>\n",
       "      <td>2025-01-04 09:59:16.158000+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.322798</td>\n",
       "      <td>1.738719</td>\n",
       "      <td>-1.584079</td>\n",
       "      <td>-47.673055</td>\n",
       "      <td>2.968027</td>\n",
       "      <td>2.028070</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.005546</td>\n",
       "      <td>-71.898714</td>\n",
       "      <td>12.878409</td>\n",
       "      <td>10.411036</td>\n",
       "      <td>-2.467374</td>\n",
       "      <td>-19.158995</td>\n",
       "      <td>471.837339</td>\n",
       "      <td>246.898066</td>\n",
       "      <td>-224.939272</td>\n",
       "      <td>-47.673055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2025_lw_human-study-on-ai-spear-phishing-campaigns.md</th>\n",
       "      <td>Human study on AI spear phishing campaigns</td>\n",
       "      <td>0.677458</td>\n",
       "      <td>2025-01-03 19:03:28.406000+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.436050</td>\n",
       "      <td>1.997053</td>\n",
       "      <td>-1.438996</td>\n",
       "      <td>-41.879381</td>\n",
       "      <td>3.160585</td>\n",
       "      <td>2.289065</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.002553</td>\n",
       "      <td>-85.223138</td>\n",
       "      <td>14.798123</td>\n",
       "      <td>12.255587</td>\n",
       "      <td>-2.542537</td>\n",
       "      <td>-17.181481</td>\n",
       "      <td>477.610894</td>\n",
       "      <td>277.590406</td>\n",
       "      <td>-200.020488</td>\n",
       "      <td>-41.879381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2025_lw_the-intelligence-curse.md</th>\n",
       "      <td>The Intelligence Curse</td>\n",
       "      <td>0.688044</td>\n",
       "      <td>2025-01-04 18:16:58.921000+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.539096</td>\n",
       "      <td>1.598036</td>\n",
       "      <td>-1.941060</td>\n",
       "      <td>-54.846207</td>\n",
       "      <td>3.480139</td>\n",
       "      <td>2.130229</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>2.610249</td>\n",
       "      <td>17.431973</td>\n",
       "      <td>11.515382</td>\n",
       "      <td>-5.916591</td>\n",
       "      <td>-33.941028</td>\n",
       "      <td>484.856104</td>\n",
       "      <td>218.930921</td>\n",
       "      <td>-265.925183</td>\n",
       "      <td>-54.846207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/lorem_ipsum.md</th>\n",
       "      <td>Lorem ipsum</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1900-01-01 00:00:00+00:00</td>\n",
       "      <td>True</td>\n",
       "      <td>2.465992</td>\n",
       "      <td>1.825440</td>\n",
       "      <td>-0.640552</td>\n",
       "      <td>-25.975428</td>\n",
       "      <td>2.463530</td>\n",
       "      <td>2.008885</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000011</td>\n",
       "      <td>-19.999573</td>\n",
       "      <td>12.736030</td>\n",
       "      <td>12.230991</td>\n",
       "      <td>-0.505038</td>\n",
       "      <td>-3.965429</td>\n",
       "      <td>490.732376</td>\n",
       "      <td>363.262540</td>\n",
       "      <td>-127.469837</td>\n",
       "      <td>-25.975428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2025_lw_the-field-of-ai-alignment-a-postmortem-and-what-to-do-about.md</th>\n",
       "      <td>The Field of AI Alignment: A Postmortem, and W...</td>\n",
       "      <td>0.925483</td>\n",
       "      <td>2025-01-01 00:42:36.538000+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.735233</td>\n",
       "      <td>1.579777</td>\n",
       "      <td>-2.155455</td>\n",
       "      <td>-57.706054</td>\n",
       "      <td>3.116857</td>\n",
       "      <td>2.056279</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000621</td>\n",
       "      <td>378.210283</td>\n",
       "      <td>12.190395</td>\n",
       "      <td>10.575907</td>\n",
       "      <td>-1.614489</td>\n",
       "      <td>-13.243940</td>\n",
       "      <td>500.521191</td>\n",
       "      <td>211.690162</td>\n",
       "      <td>-288.831029</td>\n",
       "      <td>-57.706054</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2024_openai_emails.md</th>\n",
       "      <td>OpenAI Email Archives from Musk v. Altman</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>2024-11-01 00:00:00+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.426089</td>\n",
       "      <td>2.140106</td>\n",
       "      <td>-1.285983</td>\n",
       "      <td>-37.535013</td>\n",
       "      <td>2.755313</td>\n",
       "      <td>2.184135</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000420</td>\n",
       "      <td>-95.082707</td>\n",
       "      <td>10.482502</td>\n",
       "      <td>9.602067</td>\n",
       "      <td>-0.880435</td>\n",
       "      <td>-8.399092</td>\n",
       "      <td>517.339424</td>\n",
       "      <td>323.156004</td>\n",
       "      <td>-194.183420</td>\n",
       "      <td>-37.535013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2025_lw_debating-buying-nvda-in-2019.md</th>\n",
       "      <td>debating buying NVDA in 2019</td>\n",
       "      <td>0.526975</td>\n",
       "      <td>2025-01-04 18:07:35.832000+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.498877</td>\n",
       "      <td>1.946026</td>\n",
       "      <td>-1.552851</td>\n",
       "      <td>-44.381423</td>\n",
       "      <td>2.970988</td>\n",
       "      <td>2.091860</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000716</td>\n",
       "      <td>144.210285</td>\n",
       "      <td>12.902117</td>\n",
       "      <td>9.158434</td>\n",
       "      <td>-3.743683</td>\n",
       "      <td>-29.016036</td>\n",
       "      <td>517.833801</td>\n",
       "      <td>288.011791</td>\n",
       "      <td>-229.822011</td>\n",
       "      <td>-44.381423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2025_lw_my-agi-safety-research-2024-review-25-plans.md</th>\n",
       "      <td>My AGI safety research—2024 review, ’25 plans</td>\n",
       "      <td>0.756925</td>\n",
       "      <td>2025-01-01 22:01:48.820000+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.153156</td>\n",
       "      <td>1.580539</td>\n",
       "      <td>-1.572617</td>\n",
       "      <td>-49.874378</td>\n",
       "      <td>3.112932</td>\n",
       "      <td>2.097326</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.001612</td>\n",
       "      <td>-86.930632</td>\n",
       "      <td>13.688265</td>\n",
       "      <td>11.766024</td>\n",
       "      <td>-1.922241</td>\n",
       "      <td>-14.042987</td>\n",
       "      <td>523.423819</td>\n",
       "      <td>262.369446</td>\n",
       "      <td>-261.054373</td>\n",
       "      <td>-49.874378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2024_bob_fanfic.md</th>\n",
       "      <td>Flower Crowns and Furry Mishaps by MyPalAI</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>2024-12-24 00:00:00+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.314768</td>\n",
       "      <td>2.646716</td>\n",
       "      <td>-0.668052</td>\n",
       "      <td>-20.153819</td>\n",
       "      <td>3.029083</td>\n",
       "      <td>2.704339</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000615</td>\n",
       "      <td>-35.032627</td>\n",
       "      <td>13.226252</td>\n",
       "      <td>11.086496</td>\n",
       "      <td>-2.139755</td>\n",
       "      <td>-16.178093</td>\n",
       "      <td>527.048090</td>\n",
       "      <td>420.827773</td>\n",
       "      <td>-106.220317</td>\n",
       "      <td>-20.153819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2024_gwern_reddit.md</th>\n",
       "      <td>Hardware Hedging Against Scaling Regime Shifts...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2024-08-21 00:00:00+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.907376</td>\n",
       "      <td>2.918248</td>\n",
       "      <td>-0.989128</td>\n",
       "      <td>-25.314383</td>\n",
       "      <td>3.043581</td>\n",
       "      <td>2.630350</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000549</td>\n",
       "      <td>46.703333</td>\n",
       "      <td>15.920402</td>\n",
       "      <td>14.091064</td>\n",
       "      <td>-1.829337</td>\n",
       "      <td>-11.490521</td>\n",
       "      <td>562.662201</td>\n",
       "      <td>420.227737</td>\n",
       "      <td>-142.434464</td>\n",
       "      <td>-25.314383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2025_lw_comment-on-death-and-the-gorgon.md</th>\n",
       "      <td>Comment on 'Death and the Gorgon'</td>\n",
       "      <td>0.750253</td>\n",
       "      <td>2025-01-01 06:00:24.498000+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>4.084834</td>\n",
       "      <td>2.495894</td>\n",
       "      <td>-1.588940</td>\n",
       "      <td>-38.898521</td>\n",
       "      <td>3.372985</td>\n",
       "      <td>2.744499</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.001318</td>\n",
       "      <td>-65.735003</td>\n",
       "      <td>16.383120</td>\n",
       "      <td>15.471860</td>\n",
       "      <td>-0.911260</td>\n",
       "      <td>-5.562186</td>\n",
       "      <td>616.809882</td>\n",
       "      <td>376.879962</td>\n",
       "      <td>-239.929921</td>\n",
       "      <td>-38.898521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2024_bob_fanfic2.md</th>\n",
       "      <td>Paradox's Box (Bobiverse) by Mark4man</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>2022-02-17 00:00:00+00:00</td>\n",
       "      <td>True</td>\n",
       "      <td>4.136299</td>\n",
       "      <td>3.299236</td>\n",
       "      <td>-0.837063</td>\n",
       "      <td>-20.237010</td>\n",
       "      <td>3.103517</td>\n",
       "      <td>2.757547</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.001771</td>\n",
       "      <td>-35.546726</td>\n",
       "      <td>13.576470</td>\n",
       "      <td>12.110069</td>\n",
       "      <td>-1.466401</td>\n",
       "      <td>-10.801048</td>\n",
       "      <td>620.444836</td>\n",
       "      <td>494.885355</td>\n",
       "      <td>-125.559481</td>\n",
       "      <td>-20.237010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2025_lw_what-s-the-short-timeline-plan.md</th>\n",
       "      <td>What’s the short timeline plan?</td>\n",
       "      <td>0.898161</td>\n",
       "      <td>2025-01-05 00:10:28.708000+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.970715</td>\n",
       "      <td>3.319989</td>\n",
       "      <td>-0.650725</td>\n",
       "      <td>-16.388113</td>\n",
       "      <td>3.171675</td>\n",
       "      <td>2.998089</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000231</td>\n",
       "      <td>159.805322</td>\n",
       "      <td>13.405236</td>\n",
       "      <td>17.062244</td>\n",
       "      <td>3.657008</td>\n",
       "      <td>27.280446</td>\n",
       "      <td>623.402177</td>\n",
       "      <td>521.238323</td>\n",
       "      <td>-102.163854</td>\n",
       "      <td>-16.388113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2024_lw_the-plan-2024-update.md</th>\n",
       "      <td>The Plan - 2024 Update</td>\n",
       "      <td>0.785170</td>\n",
       "      <td>2024-12-31 19:29:23.013000+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.741762</td>\n",
       "      <td>2.981632</td>\n",
       "      <td>-0.760130</td>\n",
       "      <td>-20.314768</td>\n",
       "      <td>2.872466</td>\n",
       "      <td>2.731928</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000043</td>\n",
       "      <td>70.526800</td>\n",
       "      <td>11.613052</td>\n",
       "      <td>11.740885</td>\n",
       "      <td>0.127832</td>\n",
       "      <td>1.100765</td>\n",
       "      <td>673.517207</td>\n",
       "      <td>536.693749</td>\n",
       "      <td>-136.823458</td>\n",
       "      <td>-20.314768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2025_lw_review-planecrash.md</th>\n",
       "      <td>Review: Planecrash</td>\n",
       "      <td>0.933950</td>\n",
       "      <td>2025-01-01 01:31:26.864000+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>3.475632</td>\n",
       "      <td>1.752375</td>\n",
       "      <td>-1.723257</td>\n",
       "      <td>-49.581118</td>\n",
       "      <td>3.319073</td>\n",
       "      <td>2.316245</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000083</td>\n",
       "      <td>-79.884876</td>\n",
       "      <td>14.514912</td>\n",
       "      <td>10.882781</td>\n",
       "      <td>-3.632131</td>\n",
       "      <td>-25.023443</td>\n",
       "      <td>684.699408</td>\n",
       "      <td>345.217788</td>\n",
       "      <td>-339.481621</td>\n",
       "      <td>-49.581118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>../samples/2025_lw_2024-in-ai-predictions.md</th>\n",
       "      <td>2024 in AI predictions</td>\n",
       "      <td>0.789190</td>\n",
       "      <td>2025-01-02 02:09:13.093000+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>4.635499</td>\n",
       "      <td>2.916743</td>\n",
       "      <td>-1.718757</td>\n",
       "      <td>-37.078135</td>\n",
       "      <td>3.080130</td>\n",
       "      <td>2.548945</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000238</td>\n",
       "      <td>50.615295</td>\n",
       "      <td>13.301698</td>\n",
       "      <td>12.898935</td>\n",
       "      <td>-0.402762</td>\n",
       "      <td>-3.027902</td>\n",
       "      <td>801.941370</td>\n",
       "      <td>504.596465</td>\n",
       "      <td>-297.344904</td>\n",
       "      <td>-37.078135</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                title  \\\n",
       "f                                                                                                       \n",
       "../samples/2024_trump_appointment.md                President Trump Announces Morgan Ortagus as De...   \n",
       "../samples/2024_how_to_focus.md                                                          How to Focus   \n",
       "../samples/2024_gpt4_fake_paper.md                                fake ai hoax paper made up by gpt-4   \n",
       "../samples/2025_h5n1_report.md                           CDC Report on Missouri H5N1 Serology Testing   \n",
       "../samples/2024_arxiv_meh.md                        TradingAgents: Multi-Agents LLM Financial Trad...   \n",
       "../samples/2024_lw_by-default-capital-will-matt...  By default, capital will matter more than ever...   \n",
       "../samples/2024_news_anthropic.md                   Amazon-backed Anthropic debuts AI agents that ...   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md                             The Laws of Large Numbers   \n",
       "../samples/politics_is_the_mind_killer.md                                 politics is the mind-killer   \n",
       "../samples/2024_lesswrong_slop.md                   Deontic Explorations In \"Paying To Talk To Sla...   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...     Parkinson's Law and the Ideology of Statistics   \n",
       "../samples/2024_anthropic_palintir.md               Anthropic and Palantir Partner to Bring Claude...   \n",
       "../samples/2025_lw_preference-inversion.md                                       Preference Inversion   \n",
       "../samples/2024_deliberative_alignment.md           Deliberative Alignment: Reasoning Enables Safe...   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...  The subset parity learning problem: much more ...   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...         Human study on AI spear phishing campaigns   \n",
       "../samples/2025_lw_the-intelligence-curse.md                                   The Intelligence Curse   \n",
       "../samples/lorem_ipsum.md                                                                 Lorem ipsum   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...  The Field of AI Alignment: A Postmortem, and W...   \n",
       "../samples/2024_openai_emails.md                            OpenAI Email Archives from Musk v. Altman   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md                       debating buying NVDA in 2019   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...      My AGI safety research—2024 review, ’25 plans   \n",
       "../samples/2024_bob_fanfic.md                              Flower Crowns and Furry Mishaps by MyPalAI   \n",
       "../samples/2024_gwern_reddit.md                     Hardware Hedging Against Scaling Regime Shifts...   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...                  Comment on 'Death and the Gorgon'   \n",
       "../samples/2024_bob_fanfic2.md                                  Paradox's Box (Bobiverse) by Mark4man   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...                    What’s the short timeline plan?   \n",
       "../samples/2024_lw_the-plan-2024-update.md                                     The Plan - 2024 Update   \n",
       "../samples/2025_lw_review-planecrash.md                                            Review: Planecrash   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md                                   2024 in AI predictions   \n",
       "\n",
       "                                                     novelty  \\\n",
       "f                                                              \n",
       "../samples/2024_trump_appointment.md                0.300000   \n",
       "../samples/2024_how_to_focus.md                     0.500000   \n",
       "../samples/2024_gpt4_fake_paper.md                  0.000000   \n",
       "../samples/2025_h5n1_report.md                      0.750000   \n",
       "../samples/2024_arxiv_meh.md                        0.150000   \n",
       "../samples/2024_lw_by-default-capital-will-matt...  0.906388   \n",
       "../samples/2024_news_anthropic.md                   0.500000   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md     0.540932   \n",
       "../samples/politics_is_the_mind_killer.md           0.500000   \n",
       "../samples/2024_lesswrong_slop.md                   0.100000   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...  0.677458   \n",
       "../samples/2024_anthropic_palintir.md               0.200000   \n",
       "../samples/2025_lw_preference-inversion.md          0.633321   \n",
       "../samples/2024_deliberative_alignment.md           0.600000   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...  0.697946   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...  0.677458   \n",
       "../samples/2025_lw_the-intelligence-curse.md        0.688044   \n",
       "../samples/lorem_ipsum.md                           0.000000   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...  0.925483   \n",
       "../samples/2024_openai_emails.md                    0.700000   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md  0.526975   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...  0.756925   \n",
       "../samples/2024_bob_fanfic.md                       0.300000   \n",
       "../samples/2024_gwern_reddit.md                     1.000000   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...  0.750253   \n",
       "../samples/2024_bob_fanfic2.md                      0.400000   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...  0.898161   \n",
       "../samples/2024_lw_the-plan-2024-update.md          0.785170   \n",
       "../samples/2025_lw_review-planecrash.md             0.933950   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md        0.789190   \n",
       "\n",
       "                                                                               date  \\\n",
       "f                                                                                     \n",
       "../samples/2024_trump_appointment.md                      2025-01-04 00:00:00+00:00   \n",
       "../samples/2024_how_to_focus.md                           2024-06-01 00:00:00+00:00   \n",
       "../samples/2024_gpt4_fake_paper.md                        2024-01-01 00:00:00+00:00   \n",
       "../samples/2025_h5n1_report.md                            2025-01-05 00:00:00+00:00   \n",
       "../samples/2024_arxiv_meh.md                              2024-12-28 00:00:00+00:00   \n",
       "../samples/2024_lw_by-default-capital-will-matt... 2024-12-30 19:47:19.838000+00:00   \n",
       "../samples/2024_news_anthropic.md                         2025-01-05 05:03:00+00:00   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md    2025-01-04 18:06:02.387000+00:00   \n",
       "../samples/politics_is_the_mind_killer.md                 2007-02-19 00:00:00+00:00   \n",
       "../samples/2024_lesswrong_slop.md                         2024-04-12 00:00:00+00:00   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo... 2025-01-04 22:59:57.376000+00:00   \n",
       "../samples/2024_anthropic_palintir.md                     2024-07-11 00:00:00+00:00   \n",
       "../samples/2025_lw_preference-inversion.md         2025-01-03 23:49:06.168000+00:00   \n",
       "../samples/2024_deliberative_alignment.md                 2024-12-20 00:00:00+00:00   \n",
       "../samples/2025_lw_the-subset-parity-learning-p... 2025-01-04 09:59:16.158000+00:00   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis... 2025-01-03 19:03:28.406000+00:00   \n",
       "../samples/2025_lw_the-intelligence-curse.md       2025-01-04 18:16:58.921000+00:00   \n",
       "../samples/lorem_ipsum.md                                 1900-01-01 00:00:00+00:00   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-... 2025-01-01 00:42:36.538000+00:00   \n",
       "../samples/2024_openai_emails.md                          2024-11-01 00:00:00+00:00   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md 2025-01-04 18:07:35.832000+00:00   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-... 2025-01-01 22:01:48.820000+00:00   \n",
       "../samples/2024_bob_fanfic.md                             2024-12-24 00:00:00+00:00   \n",
       "../samples/2024_gwern_reddit.md                           2024-08-21 00:00:00+00:00   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor... 2025-01-01 06:00:24.498000+00:00   \n",
       "../samples/2024_bob_fanfic2.md                            2022-02-17 00:00:00+00:00   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl... 2025-01-05 00:10:28.708000+00:00   \n",
       "../samples/2024_lw_the-plan-2024-update.md         2024-12-31 19:29:23.013000+00:00   \n",
       "../samples/2025_lw_review-planecrash.md            2025-01-01 01:31:26.864000+00:00   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md       2025-01-02 02:09:13.093000+00:00   \n",
       "\n",
       "                                                    in_training  \\\n",
       "f                                                                 \n",
       "../samples/2024_trump_appointment.md                      False   \n",
       "../samples/2024_how_to_focus.md                           False   \n",
       "../samples/2024_gpt4_fake_paper.md                        False   \n",
       "../samples/2025_h5n1_report.md                            False   \n",
       "../samples/2024_arxiv_meh.md                              False   \n",
       "../samples/2024_lw_by-default-capital-will-matt...        False   \n",
       "../samples/2024_news_anthropic.md                         False   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md           False   \n",
       "../samples/politics_is_the_mind_killer.md                  True   \n",
       "../samples/2024_lesswrong_slop.md                         False   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...        False   \n",
       "../samples/2024_anthropic_palintir.md                     False   \n",
       "../samples/2025_lw_preference-inversion.md                False   \n",
       "../samples/2024_deliberative_alignment.md                 False   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...        False   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...        False   \n",
       "../samples/2025_lw_the-intelligence-curse.md              False   \n",
       "../samples/lorem_ipsum.md                                  True   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...        False   \n",
       "../samples/2024_openai_emails.md                          False   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md        False   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...        False   \n",
       "../samples/2024_bob_fanfic.md                             False   \n",
       "../samples/2024_gwern_reddit.md                           False   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...        False   \n",
       "../samples/2024_bob_fanfic2.md                             True   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...        False   \n",
       "../samples/2024_lw_the-plan-2024-update.md                False   \n",
       "../samples/2025_lw_review-planecrash.md                   False   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md              False   \n",
       "\n",
       "                                                    train/before_mean  \\\n",
       "f                                                                       \n",
       "../samples/2024_trump_appointment.md                         3.602360   \n",
       "../samples/2024_how_to_focus.md                              2.030182   \n",
       "../samples/2024_gpt4_fake_paper.md                           2.890543   \n",
       "../samples/2025_h5n1_report.md                               2.840106   \n",
       "../samples/2024_arxiv_meh.md                                 3.329921   \n",
       "../samples/2024_lw_by-default-capital-will-matt...           2.935460   \n",
       "../samples/2024_news_anthropic.md                            2.437276   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md              2.887975   \n",
       "../samples/politics_is_the_mind_killer.md                    3.285058   \n",
       "../samples/2024_lesswrong_slop.md                            3.078750   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...           3.550214   \n",
       "../samples/2024_anthropic_palintir.md                        3.575668   \n",
       "../samples/2025_lw_preference-inversion.md                   3.358938   \n",
       "../samples/2024_deliberative_alignment.md                    3.623786   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...           3.322798   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...           3.436050   \n",
       "../samples/2025_lw_the-intelligence-curse.md                 3.539096   \n",
       "../samples/lorem_ipsum.md                                    2.465992   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...           3.735233   \n",
       "../samples/2024_openai_emails.md                             3.426089   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md           3.498877   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...           3.153156   \n",
       "../samples/2024_bob_fanfic.md                                3.314768   \n",
       "../samples/2024_gwern_reddit.md                              3.907376   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...           4.084834   \n",
       "../samples/2024_bob_fanfic2.md                               4.136299   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...           3.970715   \n",
       "../samples/2024_lw_the-plan-2024-update.md                   3.741762   \n",
       "../samples/2025_lw_review-planecrash.md                      3.475632   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md                 4.635499   \n",
       "\n",
       "                                                    train/after_mean  \\\n",
       "f                                                                      \n",
       "../samples/2024_trump_appointment.md                        1.976479   \n",
       "../samples/2024_how_to_focus.md                             1.238443   \n",
       "../samples/2024_gpt4_fake_paper.md                          1.876897   \n",
       "../samples/2025_h5n1_report.md                              1.588481   \n",
       "../samples/2024_arxiv_meh.md                                2.693382   \n",
       "../samples/2024_lw_by-default-capital-will-matt...          1.667801   \n",
       "../samples/2024_news_anthropic.md                           1.420116   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md             2.153969   \n",
       "../samples/politics_is_the_mind_killer.md                   2.430624   \n",
       "../samples/2024_lesswrong_slop.md                           1.695459   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...          2.779585   \n",
       "../samples/2024_anthropic_palintir.md                       2.444458   \n",
       "../samples/2025_lw_preference-inversion.md                  1.602391   \n",
       "../samples/2024_deliberative_alignment.md                   2.384146   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...          1.738719   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...          1.997053   \n",
       "../samples/2025_lw_the-intelligence-curse.md                1.598036   \n",
       "../samples/lorem_ipsum.md                                   1.825440   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...          1.579777   \n",
       "../samples/2024_openai_emails.md                            2.140106   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md          1.946026   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...          1.580539   \n",
       "../samples/2024_bob_fanfic.md                               2.646716   \n",
       "../samples/2024_gwern_reddit.md                             2.918248   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...          2.495894   \n",
       "../samples/2024_bob_fanfic2.md                              3.299236   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...          3.319989   \n",
       "../samples/2024_lw_the-plan-2024-update.md                  2.981632   \n",
       "../samples/2025_lw_review-planecrash.md                     1.752375   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md                2.916743   \n",
       "\n",
       "                                                    train/diff_mean  \\\n",
       "f                                                                     \n",
       "../samples/2024_trump_appointment.md                      -1.625882   \n",
       "../samples/2024_how_to_focus.md                           -0.791739   \n",
       "../samples/2024_gpt4_fake_paper.md                        -1.013646   \n",
       "../samples/2025_h5n1_report.md                            -1.251626   \n",
       "../samples/2024_arxiv_meh.md                              -0.636539   \n",
       "../samples/2024_lw_by-default-capital-will-matt...        -1.267660   \n",
       "../samples/2024_news_anthropic.md                         -1.017160   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md           -0.734006   \n",
       "../samples/politics_is_the_mind_killer.md                 -0.854434   \n",
       "../samples/2024_lesswrong_slop.md                         -1.383291   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...        -0.770629   \n",
       "../samples/2024_anthropic_palintir.md                     -1.131210   \n",
       "../samples/2025_lw_preference-inversion.md                -1.756548   \n",
       "../samples/2024_deliberative_alignment.md                 -1.239640   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...        -1.584079   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...        -1.438996   \n",
       "../samples/2025_lw_the-intelligence-curse.md              -1.941060   \n",
       "../samples/lorem_ipsum.md                                 -0.640552   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...        -2.155455   \n",
       "../samples/2024_openai_emails.md                          -1.285983   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md        -1.552851   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...        -1.572617   \n",
       "../samples/2024_bob_fanfic.md                             -0.668052   \n",
       "../samples/2024_gwern_reddit.md                           -0.989128   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...        -1.588940   \n",
       "../samples/2024_bob_fanfic2.md                            -0.837063   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...        -0.650725   \n",
       "../samples/2024_lw_the-plan-2024-update.md                -0.760130   \n",
       "../samples/2025_lw_review-planecrash.md                   -1.723257   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md              -1.718757   \n",
       "\n",
       "                                                    train/diff%_mean  \\\n",
       "f                                                                      \n",
       "../samples/2024_trump_appointment.md                      -45.133784   \n",
       "../samples/2024_how_to_focus.md                           -38.998412   \n",
       "../samples/2024_gpt4_fake_paper.md                        -35.067661   \n",
       "../samples/2025_h5n1_report.md                            -44.069677   \n",
       "../samples/2024_arxiv_meh.md                              -19.115730   \n",
       "../samples/2024_lw_by-default-capital-will-matt...        -43.184356   \n",
       "../samples/2024_news_anthropic.md                         -41.733471   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md           -25.415931   \n",
       "../samples/politics_is_the_mind_killer.md                 -26.009715   \n",
       "../samples/2024_lesswrong_slop.md                         -44.930269   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...        -21.706555   \n",
       "../samples/2024_anthropic_palintir.md                     -31.636323   \n",
       "../samples/2025_lw_preference-inversion.md                -52.294728   \n",
       "../samples/2024_deliberative_alignment.md                 -34.208421   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...        -47.673055   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...        -41.879381   \n",
       "../samples/2025_lw_the-intelligence-curse.md              -54.846207   \n",
       "../samples/lorem_ipsum.md                                 -25.975428   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...        -57.706054   \n",
       "../samples/2024_openai_emails.md                          -37.535013   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md        -44.381423   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...        -49.874378   \n",
       "../samples/2024_bob_fanfic.md                             -20.153819   \n",
       "../samples/2024_gwern_reddit.md                           -25.314383   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...        -38.898521   \n",
       "../samples/2024_bob_fanfic2.md                            -20.237010   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...        -16.388113   \n",
       "../samples/2024_lw_the-plan-2024-update.md                -20.314768   \n",
       "../samples/2025_lw_review-planecrash.md                   -49.581118   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md              -37.078135   \n",
       "\n",
       "                                                    train/before_std  \\\n",
       "f                                                                      \n",
       "../samples/2024_trump_appointment.md                        3.130189   \n",
       "../samples/2024_how_to_focus.md                             2.826893   \n",
       "../samples/2024_gpt4_fake_paper.md                          2.491279   \n",
       "../samples/2025_h5n1_report.md                              2.737036   \n",
       "../samples/2024_arxiv_meh.md                                3.075030   \n",
       "../samples/2024_lw_by-default-capital-will-matt...          2.497172   \n",
       "../samples/2024_news_anthropic.md                           2.880203   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md             2.576897   \n",
       "../samples/politics_is_the_mind_killer.md                   2.930715   \n",
       "../samples/2024_lesswrong_slop.md                           2.946400   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...          3.111395   \n",
       "../samples/2024_anthropic_palintir.md                       3.246451   \n",
       "../samples/2025_lw_preference-inversion.md                  2.745416   \n",
       "../samples/2024_deliberative_alignment.md                   3.441515   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...          2.968027   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...          3.160585   \n",
       "../samples/2025_lw_the-intelligence-curse.md                3.480139   \n",
       "../samples/lorem_ipsum.md                                   2.463530   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...          3.116857   \n",
       "../samples/2024_openai_emails.md                            2.755313   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md          2.970988   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...          3.112932   \n",
       "../samples/2024_bob_fanfic.md                               3.029083   \n",
       "../samples/2024_gwern_reddit.md                             3.043581   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...          3.372985   \n",
       "../samples/2024_bob_fanfic2.md                              3.103517   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...          3.171675   \n",
       "../samples/2024_lw_the-plan-2024-update.md                  2.872466   \n",
       "../samples/2025_lw_review-planecrash.md                     3.319073   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md                3.080130   \n",
       "\n",
       "                                                    train/after_std  ...  \\\n",
       "f                                                                    ...   \n",
       "../samples/2024_trump_appointment.md                       2.116151  ...   \n",
       "../samples/2024_how_to_focus.md                            2.380078  ...   \n",
       "../samples/2024_gpt4_fake_paper.md                         2.017982  ...   \n",
       "../samples/2025_h5n1_report.md                             2.004289  ...   \n",
       "../samples/2024_arxiv_meh.md                               2.940876  ...   \n",
       "../samples/2024_lw_by-default-capital-will-matt...         2.004676  ...   \n",
       "../samples/2024_news_anthropic.md                          2.245643  ...   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md            2.052082  ...   \n",
       "../samples/politics_is_the_mind_killer.md                  2.513867  ...   \n",
       "../samples/2024_lesswrong_slop.md                          2.349493  ...   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...         2.862761  ...   \n",
       "../samples/2024_anthropic_palintir.md                      2.673633  ...   \n",
       "../samples/2025_lw_preference-inversion.md                 1.724408  ...   \n",
       "../samples/2024_deliberative_alignment.md                  2.720267  ...   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...         2.028070  ...   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...         2.289065  ...   \n",
       "../samples/2025_lw_the-intelligence-curse.md               2.130229  ...   \n",
       "../samples/lorem_ipsum.md                                  2.008885  ...   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...         2.056279  ...   \n",
       "../samples/2024_openai_emails.md                           2.184135  ...   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md         2.091860  ...   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...         2.097326  ...   \n",
       "../samples/2024_bob_fanfic.md                              2.704339  ...   \n",
       "../samples/2024_gwern_reddit.md                            2.630350  ...   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...         2.744499  ...   \n",
       "../samples/2024_bob_fanfic2.md                             2.757547  ...   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...         2.998089  ...   \n",
       "../samples/2024_lw_the-plan-2024-update.md                 2.731928  ...   \n",
       "../samples/2025_lw_review-planecrash.md                    2.316245  ...   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md               2.548945  ...   \n",
       "\n",
       "                                                    train/diff_min  \\\n",
       "f                                                                    \n",
       "../samples/2024_trump_appointment.md                     -0.005089   \n",
       "../samples/2024_how_to_focus.md                          -0.000099   \n",
       "../samples/2024_gpt4_fake_paper.md                       -0.000804   \n",
       "../samples/2025_h5n1_report.md                           -0.002226   \n",
       "../samples/2024_arxiv_meh.md                             -0.000330   \n",
       "../samples/2024_lw_by-default-capital-will-matt...       -0.007733   \n",
       "../samples/2024_news_anthropic.md                        -0.000029   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md          -0.001162   \n",
       "../samples/politics_is_the_mind_killer.md                -0.004398   \n",
       "../samples/2024_lesswrong_slop.md                        -0.000778   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...       -0.004117   \n",
       "../samples/2024_anthropic_palintir.md                    -0.000074   \n",
       "../samples/2025_lw_preference-inversion.md               -0.003750   \n",
       "../samples/2024_deliberative_alignment.md                -0.000561   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...       -0.005546   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...       -0.002553   \n",
       "../samples/2025_lw_the-intelligence-curse.md              0.000008   \n",
       "../samples/lorem_ipsum.md                                -0.000011   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...        0.000621   \n",
       "../samples/2024_openai_emails.md                         -0.000420   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md        0.000716   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...       -0.001612   \n",
       "../samples/2024_bob_fanfic.md                            -0.000615   \n",
       "../samples/2024_gwern_reddit.md                           0.000549   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...       -0.001318   \n",
       "../samples/2024_bob_fanfic2.md                           -0.001771   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...        0.000231   \n",
       "../samples/2024_lw_the-plan-2024-update.md                0.000043   \n",
       "../samples/2025_lw_review-planecrash.md                  -0.000083   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md              0.000238   \n",
       "\n",
       "                                                    train/diff%_min  \\\n",
       "f                                                                     \n",
       "../samples/2024_trump_appointment.md                     -82.696441   \n",
       "../samples/2024_how_to_focus.md                          -40.660140   \n",
       "../samples/2024_gpt4_fake_paper.md                       -71.474606   \n",
       "../samples/2025_h5n1_report.md                           -77.666175   \n",
       "../samples/2024_arxiv_meh.md                             -61.776143   \n",
       "../samples/2024_lw_by-default-capital-will-matt...       -65.756404   \n",
       "../samples/2024_news_anthropic.md                        -71.181132   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md          -29.038092   \n",
       "../samples/politics_is_the_mind_killer.md                -65.298689   \n",
       "../samples/2024_lesswrong_slop.md                        -75.973899   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...       -90.026555   \n",
       "../samples/2024_anthropic_palintir.md                    -75.181568   \n",
       "../samples/2025_lw_preference-inversion.md               -88.162939   \n",
       "../samples/2024_deliberative_alignment.md                -29.432378   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...       -71.898714   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...       -85.223138   \n",
       "../samples/2025_lw_the-intelligence-curse.md               2.610249   \n",
       "../samples/lorem_ipsum.md                                -19.999573   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...       378.210283   \n",
       "../samples/2024_openai_emails.md                         -95.082707   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md       144.210285   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...       -86.930632   \n",
       "../samples/2024_bob_fanfic.md                            -35.032627   \n",
       "../samples/2024_gwern_reddit.md                           46.703333   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...       -65.735003   \n",
       "../samples/2024_bob_fanfic2.md                           -35.546726   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...       159.805322   \n",
       "../samples/2024_lw_the-plan-2024-update.md                70.526800   \n",
       "../samples/2025_lw_review-planecrash.md                  -79.884876   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md              50.615295   \n",
       "\n",
       "                                                    train/before_max  \\\n",
       "f                                                                      \n",
       "../samples/2024_trump_appointment.md                       17.917255   \n",
       "../samples/2024_how_to_focus.md                            13.456800   \n",
       "../samples/2024_gpt4_fake_paper.md                         11.427372   \n",
       "../samples/2025_h5n1_report.md                             11.928956   \n",
       "../samples/2024_arxiv_meh.md                               13.523319   \n",
       "../samples/2024_lw_by-default-capital-will-matt...         12.706497   \n",
       "../samples/2024_news_anthropic.md                          12.790111   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md            11.118690   \n",
       "../samples/politics_is_the_mind_killer.md                  11.889731   \n",
       "../samples/2024_lesswrong_slop.md                          15.304575   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...         12.598433   \n",
       "../samples/2024_anthropic_palintir.md                      13.271792   \n",
       "../samples/2025_lw_preference-inversion.md                 11.404502   \n",
       "../samples/2024_deliberative_alignment.md                  14.758335   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...         12.878409   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...         14.798123   \n",
       "../samples/2025_lw_the-intelligence-curse.md               17.431973   \n",
       "../samples/lorem_ipsum.md                                  12.736030   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...         12.190395   \n",
       "../samples/2024_openai_emails.md                           10.482502   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md         12.902117   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...         13.688265   \n",
       "../samples/2024_bob_fanfic.md                              13.226252   \n",
       "../samples/2024_gwern_reddit.md                            15.920402   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...         16.383120   \n",
       "../samples/2024_bob_fanfic2.md                             13.576470   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...         13.405236   \n",
       "../samples/2024_lw_the-plan-2024-update.md                 11.613052   \n",
       "../samples/2025_lw_review-planecrash.md                    14.514912   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md               13.301698   \n",
       "\n",
       "                                                    train/after_max  \\\n",
       "f                                                                     \n",
       "../samples/2024_trump_appointment.md                      11.689822   \n",
       "../samples/2024_how_to_focus.md                           13.169463   \n",
       "../samples/2024_gpt4_fake_paper.md                         9.812000   \n",
       "../samples/2025_h5n1_report.md                             9.404486   \n",
       "../samples/2024_arxiv_meh.md                              13.253697   \n",
       "../samples/2024_lw_by-default-capital-will-matt...        11.517975   \n",
       "../samples/2024_news_anthropic.md                         12.667040   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md            8.321499   \n",
       "../samples/politics_is_the_mind_killer.md                 11.973468   \n",
       "../samples/2024_lesswrong_slop.md                         12.805290   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...        12.334391   \n",
       "../samples/2024_anthropic_palintir.md                     12.891874   \n",
       "../samples/2025_lw_preference-inversion.md                 8.079728   \n",
       "../samples/2024_deliberative_alignment.md                 12.259495   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...        10.411036   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...        12.255587   \n",
       "../samples/2025_lw_the-intelligence-curse.md              11.515382   \n",
       "../samples/lorem_ipsum.md                                 12.230991   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...        10.575907   \n",
       "../samples/2024_openai_emails.md                           9.602067   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md         9.158434   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...        11.766024   \n",
       "../samples/2024_bob_fanfic.md                             11.086496   \n",
       "../samples/2024_gwern_reddit.md                           14.091064   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...        15.471860   \n",
       "../samples/2024_bob_fanfic2.md                            12.110069   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...        17.062244   \n",
       "../samples/2024_lw_the-plan-2024-update.md                11.740885   \n",
       "../samples/2025_lw_review-planecrash.md                   10.882781   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md              12.898935   \n",
       "\n",
       "                                                    train/diff_max  \\\n",
       "f                                                                    \n",
       "../samples/2024_trump_appointment.md                     -6.227433   \n",
       "../samples/2024_how_to_focus.md                          -0.287336   \n",
       "../samples/2024_gpt4_fake_paper.md                       -1.615372   \n",
       "../samples/2025_h5n1_report.md                           -2.524470   \n",
       "../samples/2024_arxiv_meh.md                             -0.269622   \n",
       "../samples/2024_lw_by-default-capital-will-matt...       -1.188522   \n",
       "../samples/2024_news_anthropic.md                        -0.123071   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md          -2.797191   \n",
       "../samples/politics_is_the_mind_killer.md                 0.083736   \n",
       "../samples/2024_lesswrong_slop.md                        -2.499285   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...       -0.264043   \n",
       "../samples/2024_anthropic_palintir.md                    -0.379918   \n",
       "../samples/2025_lw_preference-inversion.md               -3.324774   \n",
       "../samples/2024_deliberative_alignment.md                -2.498840   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...       -2.467374   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...       -2.542537   \n",
       "../samples/2025_lw_the-intelligence-curse.md             -5.916591   \n",
       "../samples/lorem_ipsum.md                                -0.505038   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...       -1.614489   \n",
       "../samples/2024_openai_emails.md                         -0.880435   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md       -3.743683   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...       -1.922241   \n",
       "../samples/2024_bob_fanfic.md                            -2.139755   \n",
       "../samples/2024_gwern_reddit.md                          -1.829337   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...       -0.911260   \n",
       "../samples/2024_bob_fanfic2.md                           -1.466401   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...        3.657008   \n",
       "../samples/2024_lw_the-plan-2024-update.md                0.127832   \n",
       "../samples/2025_lw_review-planecrash.md                  -3.632131   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md             -0.402762   \n",
       "\n",
       "                                                    train/diff%_max  \\\n",
       "f                                                                     \n",
       "../samples/2024_trump_appointment.md                     -34.756625   \n",
       "../samples/2024_how_to_focus.md                           -2.135250   \n",
       "../samples/2024_gpt4_fake_paper.md                       -14.135986   \n",
       "../samples/2025_h5n1_report.md                           -21.162542   \n",
       "../samples/2024_arxiv_meh.md                              -1.993755   \n",
       "../samples/2024_lw_by-default-capital-will-matt...        -9.353658   \n",
       "../samples/2024_news_anthropic.md                         -0.962233   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md          -25.157557   \n",
       "../samples/politics_is_the_mind_killer.md                  0.704275   \n",
       "../samples/2024_lesswrong_slop.md                        -16.330311   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...        -2.095839   \n",
       "../samples/2024_anthropic_palintir.md                     -2.862598   \n",
       "../samples/2025_lw_preference-inversion.md               -29.153170   \n",
       "../samples/2024_deliberative_alignment.md                -16.931722   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...       -19.158995   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...       -17.181481   \n",
       "../samples/2025_lw_the-intelligence-curse.md             -33.941028   \n",
       "../samples/lorem_ipsum.md                                 -3.965429   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...       -13.243940   \n",
       "../samples/2024_openai_emails.md                          -8.399092   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md       -29.016036   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...       -14.042987   \n",
       "../samples/2024_bob_fanfic.md                            -16.178093   \n",
       "../samples/2024_gwern_reddit.md                          -11.490521   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...        -5.562186   \n",
       "../samples/2024_bob_fanfic2.md                           -10.801048   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...        27.280446   \n",
       "../samples/2024_lw_the-plan-2024-update.md                 1.100765   \n",
       "../samples/2025_lw_review-planecrash.md                  -25.023443   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md              -3.027902   \n",
       "\n",
       "                                                    train/before_sum  \\\n",
       "f                                                                      \n",
       "../samples/2024_trump_appointment.md                      284.586479   \n",
       "../samples/2024_how_to_focus.md                           347.161163   \n",
       "../samples/2024_gpt4_fake_paper.md                        349.755680   \n",
       "../samples/2025_h5n1_report.md                            355.013296   \n",
       "../samples/2024_arxiv_meh.md                              386.270819   \n",
       "../samples/2024_lw_by-default-capital-will-matt...        393.351672   \n",
       "../samples/2024_news_anthropic.md                         424.085983   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md           424.532354   \n",
       "../samples/politics_is_the_mind_killer.md                 446.767846   \n",
       "../samples/2024_lesswrong_slop.md                         449.497469   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...        450.877187   \n",
       "../samples/2024_anthropic_palintir.md                     454.109781   \n",
       "../samples/2025_lw_preference-inversion.md                456.815627   \n",
       "../samples/2024_deliberative_alignment.md                 467.468432   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...        471.837339   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...        477.610894   \n",
       "../samples/2025_lw_the-intelligence-curse.md              484.856104   \n",
       "../samples/lorem_ipsum.md                                 490.732376   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...        500.521191   \n",
       "../samples/2024_openai_emails.md                          517.339424   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md        517.833801   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...        523.423819   \n",
       "../samples/2024_bob_fanfic.md                             527.048090   \n",
       "../samples/2024_gwern_reddit.md                           562.662201   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...        616.809882   \n",
       "../samples/2024_bob_fanfic2.md                            620.444836   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...        623.402177   \n",
       "../samples/2024_lw_the-plan-2024-update.md                673.517207   \n",
       "../samples/2025_lw_review-planecrash.md                   684.699408   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md              801.941370   \n",
       "\n",
       "                                                    train/after_sum  \\\n",
       "f                                                                     \n",
       "../samples/2024_trump_appointment.md                     156.141833   \n",
       "../samples/2024_how_to_focus.md                          211.773822   \n",
       "../samples/2024_gpt4_fake_paper.md                       227.104543   \n",
       "../samples/2025_h5n1_report.md                           198.560084   \n",
       "../samples/2024_arxiv_meh.md                             312.432330   \n",
       "../samples/2024_lw_by-default-capital-will-matt...       223.485284   \n",
       "../samples/2024_news_anthropic.md                        247.100184   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md          316.633504   \n",
       "../samples/politics_is_the_mind_killer.md                330.564804   \n",
       "../samples/2024_lesswrong_slop.md                        247.537046   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...       353.007281   \n",
       "../samples/2024_anthropic_palintir.md                    310.446145   \n",
       "../samples/2025_lw_preference-inversion.md               217.925137   \n",
       "../samples/2024_deliberative_alignment.md                307.554861   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...       246.898066   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...       277.590406   \n",
       "../samples/2025_lw_the-intelligence-curse.md             218.930921   \n",
       "../samples/lorem_ipsum.md                                363.262540   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...       211.690162   \n",
       "../samples/2024_openai_emails.md                         323.156004   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md       288.011791   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...       262.369446   \n",
       "../samples/2024_bob_fanfic.md                            420.827773   \n",
       "../samples/2024_gwern_reddit.md                          420.227737   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...       376.879962   \n",
       "../samples/2024_bob_fanfic2.md                           494.885355   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...       521.238323   \n",
       "../samples/2024_lw_the-plan-2024-update.md               536.693749   \n",
       "../samples/2025_lw_review-planecrash.md                  345.217788   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md             504.596465   \n",
       "\n",
       "                                                    train/diff_sum  \\\n",
       "f                                                                    \n",
       "../samples/2024_trump_appointment.md                   -128.444646   \n",
       "../samples/2024_how_to_focus.md                        -135.387340   \n",
       "../samples/2024_gpt4_fake_paper.md                     -122.651137   \n",
       "../samples/2025_h5n1_report.md                         -156.453212   \n",
       "../samples/2024_arxiv_meh.md                            -73.838489   \n",
       "../samples/2024_lw_by-default-capital-will-matt...     -169.866388   \n",
       "../samples/2024_news_anthropic.md                      -176.985800   \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md        -107.898850   \n",
       "../samples/politics_is_the_mind_killer.md              -116.203043   \n",
       "../samples/2024_lesswrong_slop.md                      -201.960423   \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...      -97.869906   \n",
       "../samples/2024_anthropic_palintir.md                  -143.663636   \n",
       "../samples/2025_lw_preference-inversion.md             -238.890490   \n",
       "../samples/2024_deliberative_alignment.md              -159.913571   \n",
       "../samples/2025_lw_the-subset-parity-learning-p...     -224.939272   \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...     -200.020488   \n",
       "../samples/2025_lw_the-intelligence-curse.md           -265.925183   \n",
       "../samples/lorem_ipsum.md                              -127.469837   \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...     -288.831029   \n",
       "../samples/2024_openai_emails.md                       -194.183420   \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md     -229.822011   \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...     -261.054373   \n",
       "../samples/2024_bob_fanfic.md                          -106.220317   \n",
       "../samples/2024_gwern_reddit.md                        -142.434464   \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...     -239.929921   \n",
       "../samples/2024_bob_fanfic2.md                         -125.559481   \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...     -102.163854   \n",
       "../samples/2024_lw_the-plan-2024-update.md             -136.823458   \n",
       "../samples/2025_lw_review-planecrash.md                -339.481621   \n",
       "../samples/2025_lw_2024-in-ai-predictions.md           -297.344904   \n",
       "\n",
       "                                                    train/diff%_sum  \n",
       "f                                                                    \n",
       "../samples/2024_trump_appointment.md                     -45.133784  \n",
       "../samples/2024_how_to_focus.md                          -38.998412  \n",
       "../samples/2024_gpt4_fake_paper.md                       -35.067661  \n",
       "../samples/2025_h5n1_report.md                           -44.069677  \n",
       "../samples/2024_arxiv_meh.md                             -19.115730  \n",
       "../samples/2024_lw_by-default-capital-will-matt...       -43.184356  \n",
       "../samples/2024_news_anthropic.md                        -41.733471  \n",
       "../samples/2025_lw_the-laws-of-large-numbers.md          -25.415931  \n",
       "../samples/politics_is_the_mind_killer.md                -26.009715  \n",
       "../samples/2024_lesswrong_slop.md                        -44.930269  \n",
       "../samples/2025_lw_parkinson-s-law-and-the-ideo...       -21.706555  \n",
       "../samples/2024_anthropic_palintir.md                    -31.636323  \n",
       "../samples/2025_lw_preference-inversion.md               -52.294728  \n",
       "../samples/2024_deliberative_alignment.md                -34.208421  \n",
       "../samples/2025_lw_the-subset-parity-learning-p...       -47.673055  \n",
       "../samples/2025_lw_human-study-on-ai-spear-phis...       -41.879381  \n",
       "../samples/2025_lw_the-intelligence-curse.md             -54.846207  \n",
       "../samples/lorem_ipsum.md                                -25.975428  \n",
       "../samples/2025_lw_the-field-of-ai-alignment-a-...       -57.706054  \n",
       "../samples/2024_openai_emails.md                         -37.535013  \n",
       "../samples/2025_lw_debating-buying-nvda-in-2019.md       -44.381423  \n",
       "../samples/2025_lw_my-agi-safety-research-2024-...       -49.874378  \n",
       "../samples/2024_bob_fanfic.md                            -20.153819  \n",
       "../samples/2024_gwern_reddit.md                          -25.314383  \n",
       "../samples/2025_lw_comment-on-death-and-the-gor...       -38.898521  \n",
       "../samples/2024_bob_fanfic2.md                           -20.237010  \n",
       "../samples/2025_lw_what-s-the-short-timeline-pl...       -16.388113  \n",
       "../samples/2024_lw_the-plan-2024-update.md               -20.314768  \n",
       "../samples/2025_lw_review-planecrash.md                  -49.581118  \n",
       "../samples/2025_lw_2024-in-ai-predictions.md             -37.078135  \n",
       "\n",
       "[30 rows x 24 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_res.sort_values('train/before_sum').set_index('f').drop(columns=['content', 'hist', 'before', 'after', 'train/before', 'train/after', 'url'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  df_res.sort_values('improvement%', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df_res.to_markdown())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DEBUG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display, HTML, Markdown\n",
    "import torch\n",
    "\n",
    "@torch.no_grad()\n",
    "def gen(model, inputs, tokenizer, clean=True):\n",
    "    s = model.generate(\n",
    "        input_ids=inputs[\"input_ids\"][None, :].to(model.device),\n",
    "        attention_mask=inputs[\"attention_mask\"][None, :].to(model.device),\n",
    "        use_cache=False,\n",
    "        max_new_tokens=100,\n",
    "        min_new_tokens=100,\n",
    "        do_sample=False,\n",
    "        early_stopping=False,\n",
    "    )\n",
    "    input_l = inputs[\"input_ids\"].shape[0]\n",
    "    tokenizer_kwargs=dict(clean_up_tokenization_spaces=clean, skip_special_tokens=clean)\n",
    "    old = tokenizer.decode(\n",
    "        s[0, :input_l][-100:], **tokenizer_kwargs\n",
    "    )\n",
    "    new = tokenizer.decode(\n",
    "        s[0, input_l:], **tokenizer_kwargs\n",
    "    )\n",
    "    s_old = \"\"+old.replace('\\n', '<br>')\n",
    "    s_new =  '<b>' + new.replace('\\n', '<br>')+ '<br><br><b/>'\n",
    "    # print(s_old, s_new)\n",
    "    display(HTML(f\"{s_old}{s_new}\"))\n",
    "    # print([old, new])\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = samples[-1]\n",
    "s = sample['content']\n",
    "first_half = s[:len(s)//2]\n",
    "second_half = s[len(s)//2:]\n",
    "ds_train = Dataset.from_dict(tokenizer([first_half]))\n",
    "ds_val = Dataset.from_dict(tokenizer([second_half]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with model.disable_adapter():\n",
    "    gen(model, ds_train.with_format('pt')[0], tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gen(model, ds_train.with_format('pt')[0], tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}