wassname/detect_bs_text
1
0
Fork 0
mirror of https://github.com/wassname/detect_bs_text.git synced 2026-07-20 12:20:23 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
3950695c702bde123a0a718c808f0ebdf90cb99c
detect_bs_text/nbs/01_detection_using_adapter_ft_split.ipynb
T
wassname 75ced83574 wip
2025-01-06 12:12:19 +08:00

221 KiB
Raw Blame History

https://github.com/huggingface/peft/blob/main/examples/fp4_finetuning/finetune_fp4_opt_bnb_peft.py

In [1]:
%reload_ext autoreload
%autoreload 2
In [2]:
from torch import optim
import lightning as pl
from matplotlib import pyplot as plt
In [3]:
from loguru import logger
import sys

# only if you want it shorter
logger.remove()
logger.add(sys.stderr, format="<level>{message}</level>", level="WARNING")
Out [3]:
1
In [4]:
import os
os.environ['CUDA_VISIBLE_DEVICES']="1"
# os.environ["CUDA_VISIBLE_DEVICES"] = "0"
import torch
import torch.nn as nn
import transformers
from datasets import load_dataset
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, AutoConfig
import numpy as np
from tqdm.auto import tqdm
import pandas as pd
import warnings
from peft import LoraConfig, get_peft_model, IA3Config
In [ ]:
In [5]:
plt.style.use('seaborn-v0_8')
torch.set_float32_matmul_precision('medium')
warnings.filterwarnings("ignore", ".*does not have many workers.*")
warnings.filterwarnings("ignore", ".*Was asked to gather along dimension 0.*")
warnings.filterwarnings("ignore", ".*There is an imbalance between your GPUs.*")
In [6]:
max_chars = 2000
In [7]:

# https://huggingface.co/collections/unsloth/llama-32-66f46afde4ca573864321a22
model_name = "unsloth/Llama-3.2-1B"
model_name = "unsloth/Llama-3.2-1B-bnb-4bit"
# Model Release Date: Sept 25, 2024
# launch date 9/25/2024 https://github.com/meta-llama/llama-models/blob/main/README.md
# https://colab.research.google.com/drive/1T5-zKWM_5OD21QHwXHiV9ixTRR7k3iB9?usp=sharing
# unsloth/Llama-3.2-3B
# Data Freshness: The pretraining data has a cutoff of December 2023.

def load_model():

    model = AutoModelForCausalLM.from_pretrained(
        model_name,
        # quantization_config=BitsAndBytesConfig(
        #     load_in_4bit=True,
        #     llm_int8_threshold=6.0,
        #     llm_int8_has_fp16_weight=False,
        #     bnb_4bit_compute_dtype=torch.float16,
        #     bnb_4bit_use_double_quant=True,
        #     bnb_4bit_quant_type="nf4",
        # ),
        torch_dtype=torch.float16,
        trust_remote_code=True,
    )


    # config = AutoConfig.from_pretrained(model_name, trust_remote_code=True,)
    # config.quantization_config['use_exllama'] = False
    # config.quantization_config['disable_exllama'] = True
    # model = AutoModelForCausalLM.from_pretrained(
    #     model_name,
    #     torch_dtype=torch.bfloat16,
    #     trust_remote_code=True,
    #     config=config,
    # )
    return model
In [8]:
base_model = load_model()
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True,)
tokenizer.pad_token = tokenizer.eos_token
Unused kwargs: ['_load_in_4bit', '_load_in_8bit', 'quant_method']. These kwargs are not used in <class 'transformers.utils.quantization_config.BitsAndBytesConfig'>.
`low_cpu_mem_usage` was None, now default to True since model is quantized.
In [9]:
def reset_model(base_model):
    # peft_config = LoraConfig(
    #     # task_type=TaskType.TOKEN_CLS, 
    #     target_modules=[ "fc2",  "Wqkv",],
    #     inference_mode=False, r=4, lora_alpha=4, 
    #     # lora_dropout=0.1, 
    #     # bias="all"
    # )
    # peft_config = IA3Config(
    #     target_modules=[ "fc2",  "Wqkv",], 
    #         feedforward_modules=["fc2"],
    #         inference_mode=False,
    # )
    peft_config = IA3Config(
        # target_modules=[ "fc2",  "Wqkv", 'out_proj', 'fc1'], 
        #     feedforward_modules=["fc2", 'fc1', 'out_proj'],
        #     inference_mode=False,
    )
    random_name = "peft_" + str(np.random.randint(0, 100000))
    model = get_peft_model(base_model, peft_config, adapter_name=random_name)
    model.config.use_cache = False
    return model

model = reset_model(base_model)
In [10]:
from bs_writing_detector.data.load_md import load_md_df
from bs_writing_detector.metrics.ppx import perplexity_compute_ds
from pathlib import Path
df = load_md_df(Path("../samples/"))

Training

In [11]:
from torch.nn import functional as F
from torch.utils.data import DataLoader, TensorDataset
from datasets import Dataset

Lightning helpers

In [ ]:
In [12]:
def eval(model, tokenizer, ds_val: Dataset):
    model.eval();
    with torch.no_grad():
        with model.disable_adapter():
            results = perplexity_compute_ds(ds=ds_val, model=model, tokenizer=tokenizer, device='cuda')['nlls'][0]
        results2 = perplexity_compute_ds(ds=ds_val, model=model, tokenizer=tokenizer, device='cuda')['nlls'][0]
    return dict(before=results, after=results2)

Train

In [13]:
from datasets import Dataset


def compute_metrics(eval_prediction):
    return {}

Trainer docs

In [14]:
from sklearn.model_selection import train_test_split

def tokenize_and_split(examples):
    l = len(tokenizer(examples).input_ids[0])
    max_len = min(l//3, max_chars) # break into at least 5
    max_len = max(max_len, 10)


    result = tokenizer(
        examples,
        add_special_tokens=False,
        truncation=True,
        stride=2, # – If set to a number along with max_length, the overflowing tokens returned will contain some tokens from the main sequence returned. The value of this argument defines the number of additional tokens.
        max_length=max_len,
        return_overflowing_tokens=True,
        return_attention_mask=True,
    )
    return result

sample = df.sample(1).iloc[0]
s = sample['content']
d = Dataset.from_dict(tokenize_and_split([s]))
d2  = d.train_test_split(test_size=0.5, seed=42)
ds_train = d2['train']
ds_val = d2['test']
ds_val
Out [14]:
Dataset({
    features: ['input_ids', 'attention_mask', 'overflow_to_sample_mapping'],
    num_rows: 2
})
In [ ]:
In [15]:
def learn_sample(sample):
    # device = 'cuda'
    # lr = 4e-3
    # epochs = 3
    # accum_steps = 1
    batch_size = 1
    verbose = False

    s = sample['content']

    d = Dataset.from_dict(tokenize_and_split([s]))
    d2  = d.train_test_split(test_size=0.5, seed=42)
    ds_train = d2['train']
    ds_val = d2['test']

    print(model.peft_config)
    model = reset_model(base_model)

    # verify that we have reset it
    print(model.peft_config)


    # eval(model, tokenizer, ds_train)

    # https://huggingface.co/docs/transformers/v4.36.1/en/main_classes/trainer#transformers.Trainer
    trainer = transformers.Trainer(
        model=model,
        train_dataset=ds_train,
        eval_dataset=ds_val,
        compute_metrics=compute_metrics, # without this it wont even give val loss
        args=transformers.TrainingArguments(
            # checkpoint='epoch',
            save_strategy='epoch',
            label_names=['labels',],
            per_device_train_batch_size=batch_size,
            # gradient_accumulation_steps=1,
            # warmup_steps=6,
            warmup_ratio=0.1,
            # max_steps=50,
            num_train_epochs=3,
            learning_rate=1e-3,
            fp16=True,
            logging_steps=1,
            output_dir="outputs",
            log_level='error',
            # do_eval=True,
            evaluation_strategy="epoch",
            eval_steps=1,
            load_best_model_at_end=True,
            
            # disable_tqdm=not verbose,
        ),
        data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),
    )
    trainer._signature_columns = ['input_ids', 'attention_mask', 'labels',]
    model.config.use_cache = False  # silence the warnings. Please re-enable for inference!
    train_output = trainer.train()

    df_hist = pd.DataFrame(trainer.state.log_history)
    df_hist_epoch = df_hist.groupby('epoch').last().drop(columns=['step'])
    df_hist_step = df_hist.set_index('step').dropna(thresh=2, axis=1)
    if verbose:
        df_hist_epoch['loss'].plot()
        plt.twinx()
        df_hist_epoch['eval_loss'].plot(c='b', label='eval')
        plt.legend()
        plt.show()


    result_train = {f'train/{k}':v for k,v in eval(model, tokenizer, ds_train).items()}
    result = eval(model, tokenizer, ds_val)
    result['hist'] = df_hist_epoch
    result.update(result_train)
    return result
In [42]:
data[0].keys()
Out [42]:
dict_keys(['before', 'after', 'hist', 'train/before', 'train/after', 'title', 'f', 'content', 'url', 'novelty', 'date', 'in_training'])
In [18]:
data = []
for i in tqdm(range(len(df))):
    sample = df.iloc[i]
    r = learn_sample(sample)
    print(sample['title'])
    print(dict(before=r['before'], after=r['after']))
    data.append(dict(**r, **sample))
  0%|          | 0/30 [00:00<?, ?it/s]
/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 2.8911, 'grad_norm': 0.8076428174972534, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 2.3384, 'grad_norm': 0.6929064989089966, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.5387918949127197, 'eval_runtime': 0.1312, 'eval_samples_per_second': 15.24, 'eval_steps_per_second': 7.62, 'epoch': 1.0}
{'loss': 2.7436, 'grad_norm': 0.7699084281921387, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.1441, 'grad_norm': 0.6051006317138672, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.5293452739715576, 'eval_runtime': 0.1328, 'eval_samples_per_second': 15.055, 'eval_steps_per_second': 7.528, 'epoch': 2.0}
{'loss': 2.5154, 'grad_norm': 0.749704122543335, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.0386, 'grad_norm': 0.5838078856468201, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.5258588790893555, 'eval_runtime': 0.1318, 'eval_samples_per_second': 15.171, 'eval_steps_per_second': 7.585, 'epoch': 3.0}
{'train_runtime': 1.3375, 'train_samples_per_second': 4.486, 'train_steps_per_second': 4.486, 'train_loss': 2.445200721422831, 'epoch': 3.0}
Anthropic and Palantir Partner to Bring Claude AI Models to AWS for U.S. Government Intelligence and Defense Operations
{'before': [11.264070510864258, 7.382840156555176, 12.170625686645508, 1.2329556941986084], 'after': [11.19461441040039, 7.479499816894531, 12.069757461547852, 0.9895272254943848]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.2976, 'grad_norm': 1.1653801202774048, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.499, 'grad_norm': 1.3143912553787231, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.3701250553131104, 'eval_runtime': 0.1588, 'eval_samples_per_second': 12.593, 'eval_steps_per_second': 6.297, 'epoch': 1.0}
{'loss': 3.0943, 'grad_norm': 1.04639732837677, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 3.1374, 'grad_norm': 1.0442923307418823, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.351206064224243, 'eval_runtime': 0.127, 'eval_samples_per_second': 15.745, 'eval_steps_per_second': 7.872, 'epoch': 2.0}
{'loss': 2.7867, 'grad_norm': 0.9014344811439514, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.9493, 'grad_norm': 0.9459804892539978, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.344069242477417, 'eval_runtime': 0.1281, 'eval_samples_per_second': 15.616, 'eval_steps_per_second': 7.808, 'epoch': 3.0}
{'train_runtime': 1.3909, 'train_samples_per_second': 4.314, 'train_steps_per_second': 4.314, 'train_loss': 3.1273703972498574, 'epoch': 3.0}
TradingAgents: Multi-Agents LLM Financial Trading Framework
{'before': [10.053862571716309, 0.6636170744895935, 8.349092483520508, 1.452925443649292, 0.9928902387619019, 8.745048522949219], 'after': [9.95648193359375, 0.47155672311782837, 8.367955207824707, 1.2811455726623535, 0.875080943107605, 8.782474517822266]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.3063, 'grad_norm': 1.185476303100586, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.3628, 'grad_norm': 1.5031108856201172, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.6609110832214355, 'eval_runtime': 0.1593, 'eval_samples_per_second': 12.556, 'eval_steps_per_second': 6.278, 'epoch': 1.0}
{'loss': 3.0717, 'grad_norm': 0.9321146011352539, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 3.0218, 'grad_norm': 1.1784974336624146, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.6481850147247314, 'eval_runtime': 0.1593, 'eval_samples_per_second': 12.552, 'eval_steps_per_second': 6.276, 'epoch': 2.0}
{'loss': 2.7501, 'grad_norm': 0.7790453433990479, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.8441, 'grad_norm': 1.0859301090240479, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.6467812061309814, 'eval_runtime': 0.1589, 'eval_samples_per_second': 12.588, 'eval_steps_per_second': 6.294, 'epoch': 3.0}
{'train_runtime': 1.6521, 'train_samples_per_second': 3.632, 'train_steps_per_second': 3.632, 'train_loss': 3.059459924697876, 'epoch': 3.0}
Flower Crowns and Furry Mishaps by MyPalAI
{'before': [7.296568393707275, 3.3268048763275146, 3.3179941177368164, 4.614715576171875, 4.787782669067383, 6.660912990570068], 'after': [7.300139904022217, 3.366487979888916, 3.34420108795166, 4.758848190307617, 4.731474876403809, 6.640608787536621]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 4.1433, 'grad_norm': 1.3135420083999634, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.8979, 'grad_norm': 1.6989140510559082, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.8832738399505615, 'eval_runtime': 0.1567, 'eval_samples_per_second': 12.767, 'eval_steps_per_second': 6.384, 'epoch': 1.0}
{'loss': 3.8552, 'grad_norm': 1.1130166053771973, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 3.445, 'grad_norm': 1.2440484762191772, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.857309103012085, 'eval_runtime': 0.1665, 'eval_samples_per_second': 12.015, 'eval_steps_per_second': 6.007, 'epoch': 2.0}
{'loss': 3.437, 'grad_norm': 1.004622459411621, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 3.2143, 'grad_norm': 1.0474002361297607, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.846127986907959, 'eval_runtime': 0.1571, 'eval_samples_per_second': 12.729, 'eval_steps_per_second': 6.365, 'epoch': 3.0}
{'train_runtime': 2.0371, 'train_samples_per_second': 2.945, 'train_steps_per_second': 2.945, 'train_loss': 3.6654568910598755, 'epoch': 3.0}
Paradox's Box (Bobiverse) by Mark4man
{'before': [8.102655410766602, 4.443521976470947, 6.978631973266602, 3.8517813682556152, 6.285520076751709], 'after': [7.454442024230957, 4.559870719909668, 7.13922643661499, 4.0717997550964355, 6.418486595153809]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.5963, 'grad_norm': 1.7606483697891235, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.8611, 'grad_norm': nan, 'learning_rate': 0.001, 'epoch': 1.0}
{'eval_loss': 4.20759391784668, 'eval_runtime': 0.1417, 'eval_samples_per_second': 14.112, 'eval_steps_per_second': 7.056, 'epoch': 1.0}
{'loss': 3.5963, 'grad_norm': 1.7607665061950684, 'learning_rate': 0.0008, 'epoch': 1.5}
{'loss': 3.8058, 'grad_norm': 5.883133411407471, 'learning_rate': 0.0006, 'epoch': 2.0}
{'eval_loss': 4.105123519897461, 'eval_runtime': 0.1429, 'eval_samples_per_second': 13.994, 'eval_steps_per_second': 6.997, 'epoch': 2.0}
{'loss': 2.6548, 'grad_norm': 1.1668059825897217, 'learning_rate': 0.0004, 'epoch': 2.5}
{'loss': 3.4369, 'grad_norm': 1.433318853378296, 'learning_rate': 0.0002, 'epoch': 3.0}
{'eval_loss': 4.070194721221924, 'eval_runtime': 0.1445, 'eval_samples_per_second': 13.837, 'eval_steps_per_second': 6.918, 'epoch': 3.0}
{'train_runtime': 2.0652, 'train_samples_per_second': 2.905, 'train_steps_per_second': 2.905, 'train_loss': 3.4918715159098306, 'epoch': 3.0}
Deliberative Alignment: Reasoning Enables Safer Language Models
{'before': [11.071215629577637, 4.1971049308776855, 5.31649112701416, 0.9296282529830933], 'after': [10.824005126953125, 4.173127174377441, 5.1760783195495605, 1.0669198036193848]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 2.8892, 'grad_norm': 1.9384897947311401, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 2.5209, 'grad_norm': 1.7461295127868652, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 2.852980613708496, 'eval_runtime': 0.1324, 'eval_samples_per_second': 15.108, 'eval_steps_per_second': 7.554, 'epoch': 1.0}
{'loss': 2.5119, 'grad_norm': 1.536733865737915, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 1.8856, 'grad_norm': 1.3685024976730347, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 2.8215484619140625, 'eval_runtime': 0.1313, 'eval_samples_per_second': 15.234, 'eval_steps_per_second': 7.617, 'epoch': 2.0}
{'loss': 2.0365, 'grad_norm': 1.2905688285827637, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 1.6189, 'grad_norm': 1.1336886882781982, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 2.816497802734375, 'eval_runtime': 0.143, 'eval_samples_per_second': 13.982, 'eval_steps_per_second': 6.991, 'epoch': 3.0}
{'train_runtime': 1.9967, 'train_samples_per_second': 3.005, 'train_steps_per_second': 3.005, 'train_loss': 2.2438440124193826, 'epoch': 3.0}
fake ai hoax paper made up by gpt-4
{'before': [7.704880714416504, 6.162439346313477, 2.8038063049316406, 6.360780715942383, 4.05305290222168, 16.83307456970215], 'after': [6.772946834564209, 5.394711494445801, 2.9839484691619873, 6.627538681030273, 3.9177215099334717, 16.98150634765625]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.8982, 'grad_norm': 1.66070556640625, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 4.0169, 'grad_norm': 1.7133033275604248, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 4.369366645812988, 'eval_runtime': 0.1559, 'eval_samples_per_second': 12.828, 'eval_steps_per_second': 6.414, 'epoch': 1.0}
{'loss': 3.5287, 'grad_norm': 1.2428947687149048, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 3.3378, 'grad_norm': 1.4541748762130737, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 4.356468677520752, 'eval_runtime': 0.158, 'eval_samples_per_second': 12.661, 'eval_steps_per_second': 6.33, 'epoch': 2.0}
{'loss': 3.0659, 'grad_norm': 0.93393474817276, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 3.0109, 'grad_norm': 1.3038973808288574, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 4.354191780090332, 'eval_runtime': 0.1594, 'eval_samples_per_second': 12.548, 'eval_steps_per_second': 6.274, 'epoch': 3.0}
{'train_runtime': 2.2731, 'train_samples_per_second': 2.64, 'train_steps_per_second': 2.64, 'train_loss': 3.476408918698629, 'epoch': 3.0}
Hardware Hedging Against Scaling Regime Shifts (self.mlscaling)
{'before': [10.767059326171875, 3.284585475921631, 2.225442886352539, 5.182804107666016], 'after': [10.917644500732422, 3.4139397144317627, 2.3915717601776123, 5.278247833251953]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 2.0168, 'grad_norm': 1.5045863389968872, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 1.9515, 'grad_norm': 1.6995381116867065, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 2.9758172035217285, 'eval_runtime': 0.1776, 'eval_samples_per_second': 11.263, 'eval_steps_per_second': 5.631, 'epoch': 1.0}
{'loss': 1.7056, 'grad_norm': 1.089555263519287, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 1.4582, 'grad_norm': 1.0138260126113892, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 2.888021469116211, 'eval_runtime': 0.1766, 'eval_samples_per_second': 11.327, 'eval_steps_per_second': 5.664, 'epoch': 2.0}
{'loss': 1.3538, 'grad_norm': 1.3632088899612427, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 1.2299, 'grad_norm': 0.939950168132782, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 2.8587775230407715, 'eval_runtime': 0.178, 'eval_samples_per_second': 11.236, 'eval_steps_per_second': 5.618, 'epoch': 3.0}
{'train_runtime': 2.458, 'train_samples_per_second': 2.441, 'train_steps_per_second': 2.441, 'train_loss': 1.619314471880595, 'epoch': 3.0}
How to Focus
{'before': [11.175151824951172, 3.43853759765625, 8.342260360717773, 8.468477249145508], 'after': [11.017436981201172, 3.291187047958374, 7.970725059509277, 8.81446647644043]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 2.9858, 'grad_norm': 2.995089054107666, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.6439, 'grad_norm': 1.58451247215271, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.429967164993286, 'eval_runtime': 0.1566, 'eval_samples_per_second': 12.768, 'eval_steps_per_second': 6.384, 'epoch': 1.0}
{'loss': 2.4293, 'grad_norm': 1.9177875518798828, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.992, 'grad_norm': 1.236825704574585, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.331143379211426, 'eval_runtime': 0.1591, 'eval_samples_per_second': 12.574, 'eval_steps_per_second': 6.287, 'epoch': 2.0}
{'loss': 1.8766, 'grad_norm': 1.3951705694198608, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.6741, 'grad_norm': 1.2922844886779785, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.304795742034912, 'eval_runtime': 0.1609, 'eval_samples_per_second': 12.431, 'eval_steps_per_second': 6.216, 'epoch': 3.0}
{'train_runtime': 2.2869, 'train_samples_per_second': 2.624, 'train_steps_per_second': 2.624, 'train_loss': 2.7669491370519004, 'epoch': 3.0}
Deontic Explorations In "Paying To Talk To Slaves"
{'before': [7.481788158416748, 3.8988535404205322, 4.159244537353516, 13.049931526184082], 'after': [4.832216262817383, 3.738274574279785, 4.002066612243652, 13.358238220214844]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 2.8747, 'grad_norm': 2.096508502960205, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.5899, 'grad_norm': 2.015336751937866, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.5862479209899902, 'eval_runtime': 0.1495, 'eval_samples_per_second': 13.375, 'eval_steps_per_second': 6.687, 'epoch': 1.0}
{'loss': 2.4316, 'grad_norm': 1.262579321861267, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.7612, 'grad_norm': 1.335434079170227, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.575761556625366, 'eval_runtime': 0.1538, 'eval_samples_per_second': 13.007, 'eval_steps_per_second': 6.503, 'epoch': 2.0}
{'loss': 1.8673, 'grad_norm': 1.1828184127807617, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.3797, 'grad_norm': 1.230667233467102, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.5691187381744385, 'eval_runtime': 0.1585, 'eval_samples_per_second': 12.622, 'eval_steps_per_second': 6.311, 'epoch': 3.0}
{'train_runtime': 2.656, 'train_samples_per_second': 2.259, 'train_steps_per_second': 2.259, 'train_loss': 2.650757590929667, 'epoch': 3.0}
By default, capital will matter more than ever after AGI
{'before': [12.33033275604248, 10.804149627685547, 5.264718532562256, 5.00636625289917, 2.185818910598755], 'after': [13.265295028686523, 10.46897029876709, 5.712345600128174, 4.91886043548584, 2.082519292831421]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.7673, 'grad_norm': 2.271059036254883, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 2.6765, 'grad_norm': 1.7324353456497192, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.623122215270996, 'eval_runtime': 0.1843, 'eval_samples_per_second': 10.85, 'eval_steps_per_second': 5.425, 'epoch': 1.0}
{'loss': 3.4631, 'grad_norm': 2.2982003688812256, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.3217, 'grad_norm': 1.8474977016448975, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.6200013160705566, 'eval_runtime': 0.186, 'eval_samples_per_second': 10.755, 'eval_steps_per_second': 5.378, 'epoch': 2.0}
{'loss': 2.9655, 'grad_norm': 1.1774643659591675, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.1688, 'grad_norm': 0.7183546423912048, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.624293565750122, 'eval_runtime': 0.1877, 'eval_samples_per_second': 10.657, 'eval_steps_per_second': 5.328, 'epoch': 3.0}
{'train_runtime': 2.6891, 'train_samples_per_second': 2.231, 'train_steps_per_second': 2.231, 'train_loss': 2.893831968307495, 'epoch': 3.0}
The Plan - 2024 Update
{'before': [11.085138320922852, 7.613076686859131, 7.271832466125488, 2.5242795944213867], 'after': [10.461780548095703, 7.329089164733887, 6.638682842254639, 2.3973302841186523]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 2.4617, 'grad_norm': 2.314183235168457, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.6791, 'grad_norm': 4.621912956237793, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 2.6804332733154297, 'eval_runtime': 0.1894, 'eval_samples_per_second': 10.557, 'eval_steps_per_second': 5.279, 'epoch': 1.0}
{'loss': 1.937, 'grad_norm': 1.5637528896331787, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.8931, 'grad_norm': 3.296773672103882, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 2.6785223484039307, 'eval_runtime': 0.1935, 'eval_samples_per_second': 10.335, 'eval_steps_per_second': 5.167, 'epoch': 2.0}
{'loss': 1.4123, 'grad_norm': 1.2823187112808228, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.5411, 'grad_norm': 1.8647823333740234, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 2.6879079341888428, 'eval_runtime': 0.1877, 'eval_samples_per_second': 10.653, 'eval_steps_per_second': 5.327, 'epoch': 3.0}
{'train_runtime': 2.7828, 'train_samples_per_second': 2.156, 'train_steps_per_second': 2.156, 'train_loss': 2.4873830676078796, 'epoch': 3.0}
Amazon-backed Anthropic debuts AI agents that can do complex tasks, racing against OpenAI, Microsoft and Google
{'before': [7.059969425201416, 7.866158485412598, 4.972960948944092, 1.4543977975845337, 11.503323554992676], 'after': [4.935206890106201, 6.333395004272461, 4.868557929992676, 1.1692737340927124, 11.316805839538574]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.4176, 'grad_norm': 2.2762768268585205, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.2375, 'grad_norm': 2.3001043796539307, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.4287538528442383, 'eval_runtime': 0.1674, 'eval_samples_per_second': 11.949, 'eval_steps_per_second': 5.975, 'epoch': 1.0}
{'loss': 2.8886, 'grad_norm': 1.398822546005249, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.4307, 'grad_norm': 1.3295927047729492, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.4019901752471924, 'eval_runtime': 0.169, 'eval_samples_per_second': 11.832, 'eval_steps_per_second': 5.916, 'epoch': 2.0}
{'loss': 2.3348, 'grad_norm': 1.320399284362793, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.0626, 'grad_norm': 1.3593965768814087, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.4008889198303223, 'eval_runtime': 0.1679, 'eval_samples_per_second': 11.912, 'eval_steps_per_second': 5.956, 'epoch': 3.0}
{'train_runtime': 2.8613, 'train_samples_per_second': 2.097, 'train_steps_per_second': 2.097, 'train_loss': 2.7286322514216104, 'epoch': 3.0}
OpenAI Email Archives from Musk v. Altman
{'before': [9.35188102722168, 3.8830721378326416, 2.591801881790161, 1.859856367111206, 10.639219284057617, 6.248669624328613], 'after': [7.7498250007629395, 3.6932098865509033, 2.136195659637451, 1.9319171905517578, 10.877782821655273, 6.625941276550293]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.5776, 'grad_norm': 4.010377883911133, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.6589, 'grad_norm': 3.6633472442626953, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.2440617084503174, 'eval_runtime': 0.1115, 'eval_samples_per_second': 17.937, 'eval_steps_per_second': 8.968, 'epoch': 1.0}
{'loss': 2.9709, 'grad_norm': 2.407945394515991, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.4008, 'grad_norm': 2.5443220138549805, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.1872379779815674, 'eval_runtime': 0.1116, 'eval_samples_per_second': 17.928, 'eval_steps_per_second': 8.964, 'epoch': 2.0}
{'loss': 2.2253, 'grad_norm': 1.7101964950561523, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 1.8499, 'grad_norm': 1.7173982858657837, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.1816787719726562, 'eval_runtime': 0.1135, 'eval_samples_per_second': 17.622, 'eval_steps_per_second': 8.811, 'epoch': 3.0}
{'train_runtime': 2.6239, 'train_samples_per_second': 2.287, 'train_steps_per_second': 2.287, 'train_loss': 2.7805453737576804, 'epoch': 3.0}
President Trump Announces Morgan Ortagus as Deputy Special Presidential Envoy for Middle East Peace
{'before': [9.416387557983398, 10.741005897521973, 2.1393628120422363, 13.070490837097168, 1.8174492120742798], 'after': [9.39091968536377, 10.194953918457031, 2.2045726776123047, 13.611124992370605, 1.771489143371582]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 2.8964, 'grad_norm': 2.3559041023254395, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 2.8855, 'grad_norm': 2.407257556915283, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.4159514904022217, 'eval_runtime': 0.1467, 'eval_samples_per_second': 13.638, 'eval_steps_per_second': 6.819, 'epoch': 1.0}
{'loss': 2.2417, 'grad_norm': 1.521421194076538, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.004, 'grad_norm': 1.6388133764266968, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.386564254760742, 'eval_runtime': 0.1444, 'eval_samples_per_second': 13.846, 'eval_steps_per_second': 6.923, 'epoch': 2.0}
{'loss': 1.5761, 'grad_norm': 1.3388304710388184, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 1.6097, 'grad_norm': 1.3109463453292847, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.395535469055176, 'eval_runtime': 0.1498, 'eval_samples_per_second': 13.355, 'eval_steps_per_second': 6.677, 'epoch': 3.0}
{'train_runtime': 2.7717, 'train_samples_per_second': 2.165, 'train_steps_per_second': 2.165, 'train_loss': 2.202237923940023, 'epoch': 3.0}
CDC Report on Missouri H5N1 Serology Testing
{'before': [9.302383422851562, 2.164090156555176, 2.530121326446533, 8.904608726501465], 'after': [7.715032577514648, 2.296508312225342, 2.2597248554229736, 10.417621612548828]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 4.6645, 'grad_norm': 2.517502546310425, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.5351, 'grad_norm': 1.9744876623153687, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 4.003051280975342, 'eval_runtime': 0.1857, 'eval_samples_per_second': 10.768, 'eval_steps_per_second': 5.384, 'epoch': 1.0}
{'loss': 4.0242, 'grad_norm': 1.5720769166946411, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.7426, 'grad_norm': 1.93308687210083, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.8951995372772217, 'eval_runtime': 0.1857, 'eval_samples_per_second': 10.772, 'eval_steps_per_second': 5.386, 'epoch': 2.0}
{'loss': 3.2325, 'grad_norm': 1.8237318992614746, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.3779, 'grad_norm': 1.5025017261505127, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.867494821548462, 'eval_runtime': 0.1954, 'eval_samples_per_second': 10.234, 'eval_steps_per_second': 5.117, 'epoch': 3.0}
{'train_runtime': 3.1729, 'train_samples_per_second': 1.891, 'train_steps_per_second': 1.891, 'train_loss': 3.429463267326355, 'epoch': 3.0}
2024 in AI predictions
{'before': [8.183303833007812, 0.7180596590042114, 8.179237365722656, 4.342537879943848, 10.570826530456543], 'after': [5.38284969329834, 0.9092058539390564, 8.8030424118042, 4.543374061584473, 10.412266731262207]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 4.0063, 'grad_norm': 2.618964672088623, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.0173, 'grad_norm': 3.8962526321411133, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 4.496792793273926, 'eval_runtime': 0.1718, 'eval_samples_per_second': 11.638, 'eval_steps_per_second': 5.819, 'epoch': 1.0}
{'loss': 3.2503, 'grad_norm': 2.181051731109619, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.2413, 'grad_norm': 1.5369327068328857, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 4.481138229370117, 'eval_runtime': 0.1743, 'eval_samples_per_second': 11.475, 'eval_steps_per_second': 5.737, 'epoch': 2.0}
{'loss': 2.4798, 'grad_norm': 1.650524377822876, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 1.8449, 'grad_norm': 1.324534296989441, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 4.495326995849609, 'eval_runtime': 0.1831, 'eval_samples_per_second': 10.923, 'eval_steps_per_second': 5.462, 'epoch': 3.0}
{'train_runtime': 3.2972, 'train_samples_per_second': 1.82, 'train_steps_per_second': 1.82, 'train_loss': 2.806656757990519, 'epoch': 3.0}
Comment on 'Death and the Gorgon'
{'before': [11.654606819152832, 3.051333427429199, 9.083542823791504, 6.945958137512207, 10.134012222290039], 'after': [12.029337882995605, 2.9569177627563477, 9.219869613647461, 7.349301338195801, 10.291199684143066]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.6214, 'grad_norm': 2.26008939743042, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.025, 'grad_norm': 2.695547580718994, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.424791097640991, 'eval_runtime': 0.1737, 'eval_samples_per_second': 11.514, 'eval_steps_per_second': 5.757, 'epoch': 1.0}
{'loss': 2.9282, 'grad_norm': 1.7743608951568604, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.0993, 'grad_norm': 1.5769550800323486, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.3480515480041504, 'eval_runtime': 0.1733, 'eval_samples_per_second': 11.538, 'eval_steps_per_second': 5.769, 'epoch': 2.0}
{'loss': 2.2028, 'grad_norm': 1.4857354164123535, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 1.6797, 'grad_norm': 1.4504806995391846, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.32250714302063, 'eval_runtime': 0.187, 'eval_samples_per_second': 10.695, 'eval_steps_per_second': 5.348, 'epoch': 3.0}
{'train_runtime': 3.2739, 'train_samples_per_second': 1.833, 'train_steps_per_second': 1.833, 'train_loss': 2.5927275021870932, 'epoch': 3.0}
debating buying NVDA in 2019
{'before': [7.8792829513549805, 5.101133346557617, 0.25373393297195435, 7.965005397796631], 'after': [4.433870792388916, 4.562448501586914, 0.215262308716774, 8.595725059509277]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.4299, 'grad_norm': 2.6792428493499756, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.3305, 'grad_norm': 2.825462818145752, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.3379433155059814, 'eval_runtime': 0.1741, 'eval_samples_per_second': 11.485, 'eval_steps_per_second': 5.742, 'epoch': 1.0}
{'loss': 2.7355, 'grad_norm': 1.6066468954086304, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.3417, 'grad_norm': 1.7739102840423584, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.325791597366333, 'eval_runtime': 0.1665, 'eval_samples_per_second': 12.009, 'eval_steps_per_second': 6.005, 'epoch': 2.0}
{'loss': 1.9831, 'grad_norm': 1.7341994047164917, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 1.8483, 'grad_norm': 1.8331834077835083, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.3339157104492188, 'eval_runtime': 0.1744, 'eval_samples_per_second': 11.469, 'eval_steps_per_second': 5.735, 'epoch': 3.0}
{'train_runtime': 3.3289, 'train_samples_per_second': 1.802, 'train_steps_per_second': 1.802, 'train_loss': 2.611490547657013, 'epoch': 3.0}
Human study on AI spear phishing campaigns
{'before': [8.617542266845703, 5.865469455718994, 5.855831146240234, 6.869799613952637, 0.7429558634757996, 5.193291664123535], 'after': [4.326836585998535, 5.962702751159668, 6.1263532638549805, 7.080767631530762, 0.37877461314201355, 5.513457298278809]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.0659, 'grad_norm': 2.3972017765045166, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.3097, 'grad_norm': 3.591853618621826, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.3992388248443604, 'eval_runtime': 0.1902, 'eval_samples_per_second': 10.513, 'eval_steps_per_second': 5.256, 'epoch': 1.0}
{'loss': 2.3208, 'grad_norm': 1.6523462533950806, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.4739, 'grad_norm': 3.74638032913208, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.375133514404297, 'eval_runtime': 0.1871, 'eval_samples_per_second': 10.691, 'eval_steps_per_second': 5.346, 'epoch': 2.0}
{'loss': 1.5714, 'grad_norm': 1.4942113161087036, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.0755, 'grad_norm': 2.089600086212158, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.3879997730255127, 'eval_runtime': 0.1873, 'eval_samples_per_second': 10.679, 'eval_steps_per_second': 5.339, 'epoch': 3.0}
{'train_runtime': 3.3319, 'train_samples_per_second': 1.801, 'train_steps_per_second': 1.801, 'train_loss': 2.469515860080719, 'epoch': 3.0}
My AGI safety research—2024 review, ’25 plans
{'before': [10.924260139465332, 4.072440147399902, 2.9389638900756836, 0.10930612683296204, 5.323551177978516], 'after': [10.128201484680176, 2.6714420318603516, 2.3213531970977783, 0.08397671580314636, 5.336966514587402]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.5354, 'grad_norm': 2.6700191497802734, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.1829, 'grad_norm': 3.94592547416687, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 4.003451824188232, 'eval_runtime': 0.1551, 'eval_samples_per_second': 12.895, 'eval_steps_per_second': 6.447, 'epoch': 1.0}
{'loss': 2.7582, 'grad_norm': 1.6923402547836304, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.2478, 'grad_norm': 3.04793643951416, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 4.013311862945557, 'eval_runtime': 0.1558, 'eval_samples_per_second': 12.838, 'eval_steps_per_second': 6.419, 'epoch': 2.0}
{'loss': 1.8756, 'grad_norm': 1.6792405843734741, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 1.7245, 'grad_norm': 2.353637933731079, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 4.035435199737549, 'eval_runtime': 0.1585, 'eval_samples_per_second': 12.617, 'eval_steps_per_second': 6.309, 'epoch': 3.0}
{'train_runtime': 3.2121, 'train_samples_per_second': 1.868, 'train_steps_per_second': 1.868, 'train_loss': 2.5540746450424194, 'epoch': 3.0}
Parkinson's Law and the Ideology of Statistics
{'before': [11.059673309326172, 0.410934716463089, 4.9378437995910645, 9.082354545593262], 'after': [11.269186019897461, 0.23422664403915405, 5.095401763916016, 9.99195671081543]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.339, 'grad_norm': 2.887451171875, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.7994, 'grad_norm': 3.2563376426696777, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.6911461353302, 'eval_runtime': 0.1747, 'eval_samples_per_second': 11.449, 'eval_steps_per_second': 5.725, 'epoch': 1.0}
{'loss': 2.4807, 'grad_norm': 2.2783405780792236, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.7528, 'grad_norm': 3.188812494277954, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.6750943660736084, 'eval_runtime': 0.1684, 'eval_samples_per_second': 11.876, 'eval_steps_per_second': 5.938, 'epoch': 2.0}
{'loss': 1.591, 'grad_norm': 2.149153470993042, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.279, 'grad_norm': 1.790748953819275, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.6802124977111816, 'eval_runtime': 0.1716, 'eval_samples_per_second': 11.652, 'eval_steps_per_second': 5.826, 'epoch': 3.0}
{'train_runtime': 3.6872, 'train_samples_per_second': 1.627, 'train_steps_per_second': 1.627, 'train_loss': 2.7069696386655173, 'epoch': 3.0}
Preference Inversion
{'before': [11.257996559143066, 2.4081733226776123, 7.490055084228516, 6.412927627563477], 'after': [12.913603782653809, 2.2612903118133545, 7.073019981384277, 8.057035446166992]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.5488, 'grad_norm': 3.1160054206848145, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 4.0606, 'grad_norm': 3.991938591003418, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.7835068702697754, 'eval_runtime': 0.2216, 'eval_samples_per_second': 9.024, 'eval_steps_per_second': 4.512, 'epoch': 1.0}
{'loss': 2.808, 'grad_norm': 2.174905300140381, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 3.2191, 'grad_norm': 7.194904804229736, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.7082245349884033, 'eval_runtime': 0.2222, 'eval_samples_per_second': 8.999, 'eval_steps_per_second': 4.5, 'epoch': 2.0}
{'loss': 2.043, 'grad_norm': 1.584335207939148, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.8653, 'grad_norm': 2.442568778991699, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.646120548248291, 'eval_runtime': 0.2322, 'eval_samples_per_second': 8.613, 'eval_steps_per_second': 4.307, 'epoch': 3.0}
{'train_runtime': 4.3763, 'train_samples_per_second': 1.371, 'train_steps_per_second': 1.371, 'train_loss': 3.09080179532369, 'epoch': 3.0}
Review: Planecrash
{'before': [8.528902053833008, 13.35799789428711, 8.998847961425781, 1.961258888244629, 0.19278351962566376], 'after': [9.244955062866211, 12.997112274169922, 4.955875396728516, 0.008964293636381626, 0.050971198827028275]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.7415, 'grad_norm': 2.612143039703369, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 2.835, 'grad_norm': 3.254910469055176, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.560338258743286, 'eval_runtime': 0.1714, 'eval_samples_per_second': 11.669, 'eval_steps_per_second': 5.835, 'epoch': 1.0}
{'loss': 2.8215, 'grad_norm': 2.334237575531006, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 1.6291, 'grad_norm': 1.8375204801559448, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.4723968505859375, 'eval_runtime': 0.171, 'eval_samples_per_second': 11.698, 'eval_steps_per_second': 5.849, 'epoch': 2.0}
{'loss': 1.9053, 'grad_norm': 2.1019699573516846, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 1.1471, 'grad_norm': 1.539451241493225, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.4505653381347656, 'eval_runtime': 0.176, 'eval_samples_per_second': 11.361, 'eval_steps_per_second': 5.681, 'epoch': 3.0}
{'train_runtime': 3.9229, 'train_samples_per_second': 1.529, 'train_steps_per_second': 1.529, 'train_loss': 2.346584419409434, 'epoch': 3.0}
The Field of AI Alignment: A Postmortem, and What To Do About It
{'before': [10.715858459472656, 2.8538293838500977, 9.619916915893555, 2.8236703872680664, 1.3537986278533936, 9.357948303222656], 'after': [17.13212776184082, 2.7473714351654053, 9.264034271240234, 3.5286686420440674, 0.48276376724243164, 9.244832992553711]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.3063, 'grad_norm': 3.2994930744171143, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.6343, 'grad_norm': 3.1291298866271973, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.973965644836426, 'eval_runtime': 0.1735, 'eval_samples_per_second': 11.525, 'eval_steps_per_second': 5.763, 'epoch': 1.0}
{'loss': 2.4674, 'grad_norm': 2.477537155151367, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.3556, 'grad_norm': 2.0268757343292236, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.917025566101074, 'eval_runtime': 0.1726, 'eval_samples_per_second': 11.589, 'eval_steps_per_second': 5.795, 'epoch': 2.0}
{'loss': 1.5868, 'grad_norm': 1.8040292263031006, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 1.7624, 'grad_norm': 1.7323975563049316, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.9284799098968506, 'eval_runtime': 0.1773, 'eval_samples_per_second': 11.28, 'eval_steps_per_second': 5.64, 'epoch': 3.0}
{'train_runtime': 3.5944, 'train_samples_per_second': 1.669, 'train_steps_per_second': 1.669, 'train_loss': 2.518795907497406, 'epoch': 3.0}
The Intelligence Curse
{'before': [11.011114120483398, 0.6101000308990479, 8.701272964477539, 2.934598207473755, 10.553529739379883], 'after': [11.304214477539062, 0.4481358230113983, 7.420818328857422, 2.2388195991516113, 10.681926727294922]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 2.983, 'grad_norm': 3.2141897678375244, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.3046, 'grad_norm': 3.5254054069519043, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 2.876291275024414, 'eval_runtime': 0.1805, 'eval_samples_per_second': 11.079, 'eval_steps_per_second': 5.54, 'epoch': 1.0}
{'loss': 2.1398, 'grad_norm': 2.1053242683410645, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.0996, 'grad_norm': 1.6926547288894653, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 2.890993595123291, 'eval_runtime': 0.1859, 'eval_samples_per_second': 10.761, 'eval_steps_per_second': 5.381, 'epoch': 2.0}
{'loss': 1.3094, 'grad_norm': 1.8891417980194092, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 1.5913, 'grad_norm': 1.575945496559143, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 2.9214653968811035, 'eval_runtime': 0.1873, 'eval_samples_per_second': 10.677, 'eval_steps_per_second': 5.338, 'epoch': 3.0}
{'train_runtime': 3.8221, 'train_samples_per_second': 1.57, 'train_steps_per_second': 1.57, 'train_loss': 2.2379422187805176, 'epoch': 3.0}
The Laws of Large Numbers
{'before': [11.152969360351562, 12.381094932556152, 6.883550643920898, 5.000158309936523], 'after': [10.98746109008789, 12.829581260681152, 6.23975944519043, 4.902842998504639]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.4465, 'grad_norm': 3.0251810550689697, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 2.4625, 'grad_norm': 3.3443939685821533, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.2333736419677734, 'eval_runtime': 0.1783, 'eval_samples_per_second': 11.22, 'eval_steps_per_second': 5.61, 'epoch': 1.0}
{'loss': 2.6116, 'grad_norm': 2.0492935180664062, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 1.4007, 'grad_norm': 1.8274493217468262, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.223329782485962, 'eval_runtime': 0.1792, 'eval_samples_per_second': 11.162, 'eval_steps_per_second': 5.581, 'epoch': 2.0}
{'loss': 1.7269, 'grad_norm': 1.6054573059082031, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 0.9109, 'grad_norm': 1.4826263189315796, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.2280447483062744, 'eval_runtime': 0.1846, 'eval_samples_per_second': 10.834, 'eval_steps_per_second': 5.417, 'epoch': 3.0}
{'train_runtime': 3.8591, 'train_samples_per_second': 1.555, 'train_steps_per_second': 1.555, 'train_loss': 2.0931734144687653, 'epoch': 3.0}
The subset parity learning problem: much more than you wanted to know
{'before': [6.899591445922852, 0.8442123532295227, 0.49031174182891846, 9.700416564941406], 'after': [5.6730499267578125, 0.8620076775550842, 0.47212398052215576, 10.04554271697998]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 4.1168, 'grad_norm': 3.2236294746398926, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.8383, 'grad_norm': 3.7380311489105225, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 2.892657995223999, 'eval_runtime': 0.1945, 'eval_samples_per_second': 10.282, 'eval_steps_per_second': 5.141, 'epoch': 1.0}
{'loss': 3.2994, 'grad_norm': 2.064343214035034, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.7109, 'grad_norm': 2.8733603954315186, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 2.893580198287964, 'eval_runtime': 0.1934, 'eval_samples_per_second': 10.341, 'eval_steps_per_second': 5.17, 'epoch': 2.0}
{'loss': 2.3681, 'grad_norm': 2.0598092079162598, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.1504, 'grad_norm': 2.0436267852783203, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 2.8934695720672607, 'eval_runtime': 0.1953, 'eval_samples_per_second': 10.239, 'eval_steps_per_second': 5.12, 'epoch': 3.0}
{'train_runtime': 4.0349, 'train_samples_per_second': 1.487, 'train_steps_per_second': 1.487, 'train_loss': 3.080639600753784, 'epoch': 3.0}
What’s the short timeline plan?
{'before': [9.168852806091309, 3.9152371883392334, 6.799325942993164, 6.304879188537598], 'after': [4.647651195526123, 4.194304466247559, 7.104080677032471, 6.2407636642456055]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 2.9721, 'grad_norm': 6.88749885559082, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 2.8245, 'grad_norm': 6.573474407196045, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 2.963982105255127, 'eval_runtime': 0.2374, 'eval_samples_per_second': 8.426, 'eval_steps_per_second': 4.213, 'epoch': 1.0}
{'loss': 2.7296, 'grad_norm': 6.738276481628418, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.2587, 'grad_norm': 5.604180812835693, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 2.7031450271606445, 'eval_runtime': 0.2389, 'eval_samples_per_second': 8.372, 'eval_steps_per_second': 4.186, 'epoch': 2.0}
{'loss': 2.0789, 'grad_norm': 5.735146522521973, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 1.7098, 'grad_norm': 3.3593287467956543, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 2.540477991104126, 'eval_runtime': 0.2464, 'eval_samples_per_second': 8.117, 'eval_steps_per_second': 4.059, 'epoch': 3.0}
{'train_runtime': 4.1935, 'train_samples_per_second': 1.431, 'train_steps_per_second': 1.431, 'train_loss': 2.4289310773213706, 'epoch': 3.0}
Lorem ipsum
{'before': [13.19483470916748, 4.822676658630371, 4.694229602813721, 0.15062260627746582, 0.009142794646322727], 'after': [12.946345329284668, 4.517403602600098, 5.635333061218262, 0.7315716743469238, 0.001257344032637775]}

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(

{'loss': 3.3232, 'grad_norm': 2.512242555618286, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 3.0244, 'grad_norm': 2.4470341205596924, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.506387233734131, 'eval_runtime': 0.1841, 'eval_samples_per_second': 10.862, 'eval_steps_per_second': 5.431, 'epoch': 1.0}
{'loss': 2.4132, 'grad_norm': 1.939867615699768, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 1.8838, 'grad_norm': 2.473356246948242, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.5382893085479736, 'eval_runtime': 0.1772, 'eval_samples_per_second': 11.287, 'eval_steps_per_second': 5.644, 'epoch': 2.0}
{'loss': 1.4453, 'grad_norm': 1.9782445430755615, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 1.3757, 'grad_norm': 1.529227375984192, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.5465643405914307, 'eval_runtime': 0.1858, 'eval_samples_per_second': 10.765, 'eval_steps_per_second': 5.382, 'epoch': 3.0}
{'train_runtime': 3.7878, 'train_samples_per_second': 1.584, 'train_steps_per_second': 1.584, 'train_loss': 2.2442721724510193, 'epoch': 3.0}
politics is the mind-killer
{'before': [11.374530792236328, 3.393611192703247, 13.047080993652344, 3.0926449298858643, 6.4303483963012695, 8.23534870147705], 'after': [11.683854103088379, 3.5927183628082275, 13.514053344726562, 3.821317672729492, 7.558355808258057, 7.392338275909424]}
In [19]:
# example training
df_hist = data[-1]['hist']#.groupby('epoch').last().dropna(axis=1).drop(columns=['step'])
df_hist['loss'].plot(label='train')
plt.twinx()
df_hist['eval_loss'].plot(c='b', label='eval')
plt.legend()
plt.show()
In [21]:
# df_hist['learning_rate'].plot(logy=True)

Perplexity

Perplexity measures how well a language model predicts a text sample. Lower is better

It’s calculated as the average number of bits per word a model needs to represent the same

https://huggingface.co/docs/transformers/perplexity https://thegradient.pub/understanding-evaluation-metrics-for-language-models/

The improvement column, is perplexity decrease

In [44]:
# df_res = pd.DataFrame(data)
In [84]:
df_res = pd.DataFrame(data)
for stat in ['mean', 'std', 'min', 'max', 'sum']:
    agg = getattr(np, stat)
    df_res[f'train/before_{stat}'] = df_res['train/before'].apply(lambda x: agg(x))
    df_res[f'train/after_{stat}'] = df_res['train/after'].apply(lambda x: agg(x)) 

    # df_res[f'before_{stat}'] = df_res['before'].apply(lambda x: agg(x))
    # df_res[f'after_{stat}'] = df_res['after'].apply(lambda x: agg(x))

    # df_res[f"diff_{stat}"] = - df_res[f'before_{stat}'] + df_res[f'after_{stat}']
    # df_res[f"diff%_{stat}"] = df_res[f"diff_{stat}"] / df_res[f'before_{stat}'] * 100

    df_res[f"train/diff_{stat}"] = -df_res[f'train/before_{stat}'] + df_res[f'train/after_{stat}']
    df_res[f"train/diff%_{stat}"] = df_res[f"train/diff_{stat}"] / df_res[f'train/before_{stat}'] * 100

r = df_res.select_dtypes(include=np.number).corr()['novelty'].sort_values()
print(stat)
display(r)

sum

train/diff_sum      -0.474407
train/diff_mean     -0.383682
train/diff%_sum     -0.253703
train/diff%_mean    -0.253703
train/diff_std      -0.226926
train/diff%_std     -0.184180
train/diff_min      -0.082520
train/after_std      0.016230
train/before_max     0.026255
train/diff%_max      0.056223
train/diff_max       0.059453
train/after_mean     0.073286
train/after_max      0.082784
train/before_min     0.126469
train/after_min      0.176288
train/after_sum      0.182927
train/before_std     0.281097
train/diff%_min      0.342018
train/before_mean    0.388784
train/before_sum     0.455190
novelty              1.000000
Name: novelty, dtype: float64
In [89]:
# also try with only new stuff, no  less wrong, only less wrong...
m = df_res.url.str.contains('lesswrong').fillna(False)
r = df_res[m].select_dtypes(include=np.number).corr()['novelty'].sort_values()
print('only lesswrong')
display(r)

r = df_res[~m].select_dtypes(include=np.number).corr()['novelty'].sort_values()
print('without lesswrong')
display(r)

r = df_res[~df_res.in_training].select_dtypes(include=np.number).corr()['novelty'].sort_values()
print('only new')
display(r)
only lesswrong

/tmp/ipykernel_1883910/1966928096.py:2: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`
  m = df_res.url.str.contains('lesswrong').fillna(False)

train/diff_sum      -0.247629
train/diff_mean     -0.159304
train/before_max    -0.077736
train/diff_std      -0.041644
train/diff%_sum     -0.034815
train/diff%_mean    -0.034815
train/before_min    -0.007299
train/after_min     -0.005681
train/diff%_std     -0.001165
train/diff_min       0.007247
train/after_std      0.111126
train/after_mean     0.168236
train/after_max      0.178731
train/before_std     0.202097
train/after_sum      0.261667
train/diff_max       0.264107
train/diff%_max      0.274232
train/diff%_min      0.318505
train/before_mean    0.390433
train/before_sum     0.418176
novelty              1.000000
Name: novelty, dtype: float64
without lesswrong

train/diff_sum      -0.524070
train/diff_mean     -0.289291
train/diff%_sum     -0.212006
train/diff%_mean    -0.212006
train/diff_std      -0.205211
train/diff%_max     -0.173305
train/diff_max      -0.153685
train/diff%_std     -0.135741
train/diff_min       0.065099
train/after_sum      0.075106
train/after_mean     0.077967
train/before_min     0.133288
train/after_std      0.148123
train/before_mean    0.209628
train/before_sum     0.217732
train/after_max      0.219580
train/before_max     0.319630
train/before_std     0.337727
train/after_min      0.384317
train/diff%_min      0.484895
novelty              1.000000
Name: novelty, dtype: float64
only new

train/diff_sum      -0.445037
train/diff_mean     -0.296686
train/diff_std      -0.175885
train/diff%_sum     -0.169818
train/diff%_mean    -0.169818
train/diff%_std     -0.149809
train/diff_min      -0.063278
train/after_std     -0.029953
train/before_max    -0.012773
train/after_mean     0.102790
train/after_max      0.116678
train/before_min     0.129897
train/diff%_max      0.131804
train/diff_max       0.133860
train/before_std     0.172009
train/after_min      0.238759
train/after_sum      0.306655
train/before_mean    0.357417
train/diff%_min      0.367850
train/before_sum     0.536616
novelty              1.000000
Name: novelty, dtype: float64
In [86]:
main_metric = 'train/diff_sum'
df_res[['f', main_metric, 'novelty', 'in_training']].sort_values( main_metric)
Out [86]:
f train/diff_sum novelty in_training
22 ../samples/2025_lw_review-planecrash.md -339.481621 0.933950 False
15 ../samples/2025_lw_2024-in-ai-predictions.md -297.344904 0.789190 False
23 ../samples/2025_lw_the-field-of-ai-alignment-a... -288.831029 0.925483 False
24 ../samples/2025_lw_the-intelligence-curse.md -265.925183 0.688044 False
19 ../samples/2025_lw_my-agi-safety-research-2024... -261.054373 0.756925 False
16 ../samples/2025_lw_comment-on-death-and-the-go... -239.929921 0.750253 False
21 ../samples/2025_lw_preference-inversion.md -238.890490 0.633321 False
17 ../samples/2025_lw_debating-buying-nvda-in-201... -229.822011 0.526975 False
26 ../samples/2025_lw_the-subset-parity-learning-... -224.939272 0.697946 False
8 ../samples/2024_lesswrong_slop.md -201.960423 0.100000 False
18 ../samples/2025_lw_human-study-on-ai-spear-phi... -200.020488 0.677458 False
12 ../samples/2024_openai_emails.md -194.183420 0.700000 False
11 ../samples/2024_news_anthropic.md -176.985800 0.500000 False
9 ../samples/2024_lw_by-default-capital-will-mat... -169.866388 0.906388 False
4 ../samples/2024_deliberative_alignment.md -159.913571 0.600000 False
14 ../samples/2025_h5n1_report.md -156.453212 0.750000 False
0 ../samples/2024_anthropic_palintir.md -143.663636 0.200000 False
6 ../samples/2024_gwern_reddit.md -142.434464 1.000000 False
10 ../samples/2024_lw_the-plan-2024-update.md -136.823458 0.785170 False
7 ../samples/2024_how_to_focus.md -135.387340 0.500000 False
13 ../samples/2024_trump_appointment.md -128.444646 0.300000 False
28 ../samples/lorem_ipsum.md -127.469837 0.000000 True
3 ../samples/2024_bob_fanfic2.md -125.559481 0.400000 True
5 ../samples/2024_gpt4_fake_paper.md -122.651137 0.000000 False
29 ../samples/politics_is_the_mind_killer.md -116.203043 0.500000 True
25 ../samples/2025_lw_the-laws-of-large-numbers.md -107.898850 0.540932 False
2 ../samples/2024_bob_fanfic.md -106.220317 0.300000 False
27 ../samples/2025_lw_what-s-the-short-timeline-p... -102.163854 0.898161 False
20 ../samples/2025_lw_parkinson-s-law-and-the-ide... -97.869906 0.677458 False
1 ../samples/2024_arxiv_meh.md -73.838489 0.150000 False
In [87]:
df_res.sort_values('train/before_sum').set_index('f').drop(columns=['content', 'hist', 'before', 'after', 'train/before', 'train/after', 'url'])
Out [87]:
title novelty date in_training train/before_mean train/after_mean train/diff_mean train/diff%_mean train/before_std train/after_std ... train/diff_min train/diff%_min train/before_max train/after_max train/diff_max train/diff%_max train/before_sum train/after_sum train/diff_sum train/diff%_sum
f
../samples/2024_trump_appointment.md President Trump Announces Morgan Ortagus as De... 0.300000 2025-01-04 00:00:00+00:00 False 3.602360 1.976479 -1.625882 -45.133784 3.130189 2.116151 ... -0.005089 -82.696441 17.917255 11.689822 -6.227433 -34.756625 284.586479 156.141833 -128.444646 -45.133784
../samples/2024_how_to_focus.md How to Focus 0.500000 2024-06-01 00:00:00+00:00 False 2.030182 1.238443 -0.791739 -38.998412 2.826893 2.380078 ... -0.000099 -40.660140 13.456800 13.169463 -0.287336 -2.135250 347.161163 211.773822 -135.387340 -38.998412
../samples/2024_gpt4_fake_paper.md fake ai hoax paper made up by gpt-4 0.000000 2024-01-01 00:00:00+00:00 False 2.890543 1.876897 -1.013646 -35.067661 2.491279 2.017982 ... -0.000804 -71.474606 11.427372 9.812000 -1.615372 -14.135986 349.755680 227.104543 -122.651137 -35.067661
../samples/2025_h5n1_report.md CDC Report on Missouri H5N1 Serology Testing 0.750000 2025-01-05 00:00:00+00:00 False 2.840106 1.588481 -1.251626 -44.069677 2.737036 2.004289 ... -0.002226 -77.666175 11.928956 9.404486 -2.524470 -21.162542 355.013296 198.560084 -156.453212 -44.069677
../samples/2024_arxiv_meh.md TradingAgents: Multi-Agents LLM Financial Trad... 0.150000 2024-12-28 00:00:00+00:00 False 3.329921 2.693382 -0.636539 -19.115730 3.075030 2.940876 ... -0.000330 -61.776143 13.523319 13.253697 -0.269622 -1.993755 386.270819 312.432330 -73.838489 -19.115730
../samples/2024_lw_by-default-capital-will-matter-more-than-ever-after-agi.md By default, capital will matter more than ever... 0.906388 2024-12-30 19:47:19.838000+00:00 False 2.935460 1.667801 -1.267660 -43.184356 2.497172 2.004676 ... -0.007733 -65.756404 12.706497 11.517975 -1.188522 -9.353658 393.351672 223.485284 -169.866388 -43.184356
../samples/2024_news_anthropic.md Amazon-backed Anthropic debuts AI agents that ... 0.500000 2025-01-05 05:03:00+00:00 False 2.437276 1.420116 -1.017160 -41.733471 2.880203 2.245643 ... -0.000029 -71.181132 12.790111 12.667040 -0.123071 -0.962233 424.085983 247.100184 -176.985800 -41.733471
../samples/2025_lw_the-laws-of-large-numbers.md The Laws of Large Numbers 0.540932 2025-01-04 18:06:02.387000+00:00 False 2.887975 2.153969 -0.734006 -25.415931 2.576897 2.052082 ... -0.001162 -29.038092 11.118690 8.321499 -2.797191 -25.157557 424.532354 316.633504 -107.898850 -25.415931
../samples/politics_is_the_mind_killer.md politics is the mind-killer 0.500000 2007-02-19 00:00:00+00:00 True 3.285058 2.430624 -0.854434 -26.009715 2.930715 2.513867 ... -0.004398 -65.298689 11.889731 11.973468 0.083736 0.704275 446.767846 330.564804 -116.203043 -26.009715
../samples/2024_lesswrong_slop.md Deontic Explorations In "Paying To Talk To Sla... 0.100000 2024-04-12 00:00:00+00:00 False 3.078750 1.695459 -1.383291 -44.930269 2.946400 2.349493 ... -0.000778 -75.973899 15.304575 12.805290 -2.499285 -16.330311 449.497469 247.537046 -201.960423 -44.930269
../samples/2025_lw_parkinson-s-law-and-the-ideology-of-statistics-1.md Parkinson's Law and the Ideology of Statistics 0.677458 2025-01-04 22:59:57.376000+00:00 False 3.550214 2.779585 -0.770629 -21.706555 3.111395 2.862761 ... -0.004117 -90.026555 12.598433 12.334391 -0.264043 -2.095839 450.877187 353.007281 -97.869906 -21.706555
../samples/2024_anthropic_palintir.md Anthropic and Palantir Partner to Bring Claude... 0.200000 2024-07-11 00:00:00+00:00 False 3.575668 2.444458 -1.131210 -31.636323 3.246451 2.673633 ... -0.000074 -75.181568 13.271792 12.891874 -0.379918 -2.862598 454.109781 310.446145 -143.663636 -31.636323
../samples/2025_lw_preference-inversion.md Preference Inversion 0.633321 2025-01-03 23:49:06.168000+00:00 False 3.358938 1.602391 -1.756548 -52.294728 2.745416 1.724408 ... -0.003750 -88.162939 11.404502 8.079728 -3.324774 -29.153170 456.815627 217.925137 -238.890490 -52.294728
../samples/2024_deliberative_alignment.md Deliberative Alignment: Reasoning Enables Safe... 0.600000 2024-12-20 00:00:00+00:00 False 3.623786 2.384146 -1.239640 -34.208421 3.441515 2.720267 ... -0.000561 -29.432378 14.758335 12.259495 -2.498840 -16.931722 467.468432 307.554861 -159.913571 -34.208421
../samples/2025_lw_the-subset-parity-learning-problem-much-more-than-you-wanted.md The subset parity learning problem: much more ... 0.697946 2025-01-04 09:59:16.158000+00:00 False 3.322798 1.738719 -1.584079 -47.673055 2.968027 2.028070 ... -0.005546 -71.898714 12.878409 10.411036 -2.467374 -19.158995 471.837339 246.898066 -224.939272 -47.673055
../samples/2025_lw_human-study-on-ai-spear-phishing-campaigns.md Human study on AI spear phishing campaigns 0.677458 2025-01-03 19:03:28.406000+00:00 False 3.436050 1.997053 -1.438996 -41.879381 3.160585 2.289065 ... -0.002553 -85.223138 14.798123 12.255587 -2.542537 -17.181481 477.610894 277.590406 -200.020488 -41.879381
../samples/2025_lw_the-intelligence-curse.md The Intelligence Curse 0.688044 2025-01-04 18:16:58.921000+00:00 False 3.539096 1.598036 -1.941060 -54.846207 3.480139 2.130229 ... 0.000008 2.610249 17.431973 11.515382 -5.916591 -33.941028 484.856104 218.930921 -265.925183 -54.846207
../samples/lorem_ipsum.md Lorem ipsum 0.000000 1900-01-01 00:00:00+00:00 True 2.465992 1.825440 -0.640552 -25.975428 2.463530 2.008885 ... -0.000011 -19.999573 12.736030 12.230991 -0.505038 -3.965429 490.732376 363.262540 -127.469837 -25.975428
../samples/2025_lw_the-field-of-ai-alignment-a-postmortem-and-what-to-do-about.md The Field of AI Alignment: A Postmortem, and W... 0.925483 2025-01-01 00:42:36.538000+00:00 False 3.735233 1.579777 -2.155455 -57.706054 3.116857 2.056279 ... 0.000621 378.210283 12.190395 10.575907 -1.614489 -13.243940 500.521191 211.690162 -288.831029 -57.706054
../samples/2024_openai_emails.md OpenAI Email Archives from Musk v. Altman 0.700000 2024-11-01 00:00:00+00:00 False 3.426089 2.140106 -1.285983 -37.535013 2.755313 2.184135 ... -0.000420 -95.082707 10.482502 9.602067 -0.880435 -8.399092 517.339424 323.156004 -194.183420 -37.535013
../samples/2025_lw_debating-buying-nvda-in-2019.md debating buying NVDA in 2019 0.526975 2025-01-04 18:07:35.832000+00:00 False 3.498877 1.946026 -1.552851 -44.381423 2.970988 2.091860 ... 0.000716 144.210285 12.902117 9.158434 -3.743683 -29.016036 517.833801 288.011791 -229.822011 -44.381423
../samples/2025_lw_my-agi-safety-research-2024-review-25-plans.md My AGI safety research—2024 review, ’25 plans 0.756925 2025-01-01 22:01:48.820000+00:00 False 3.153156 1.580539 -1.572617 -49.874378 3.112932 2.097326 ... -0.001612 -86.930632 13.688265 11.766024 -1.922241 -14.042987 523.423819 262.369446 -261.054373 -49.874378
../samples/2024_bob_fanfic.md Flower Crowns and Furry Mishaps by MyPalAI 0.300000 2024-12-24 00:00:00+00:00 False 3.314768 2.646716 -0.668052 -20.153819 3.029083 2.704339 ... -0.000615 -35.032627 13.226252 11.086496 -2.139755 -16.178093 527.048090 420.827773 -106.220317 -20.153819
../samples/2024_gwern_reddit.md Hardware Hedging Against Scaling Regime Shifts... 1.000000 2024-08-21 00:00:00+00:00 False 3.907376 2.918248 -0.989128 -25.314383 3.043581 2.630350 ... 0.000549 46.703333 15.920402 14.091064 -1.829337 -11.490521 562.662201 420.227737 -142.434464 -25.314383
../samples/2025_lw_comment-on-death-and-the-gorgon.md Comment on 'Death and the Gorgon' 0.750253 2025-01-01 06:00:24.498000+00:00 False 4.084834 2.495894 -1.588940 -38.898521 3.372985 2.744499 ... -0.001318 -65.735003 16.383120 15.471860 -0.911260 -5.562186 616.809882 376.879962 -239.929921 -38.898521
../samples/2024_bob_fanfic2.md Paradox's Box (Bobiverse) by Mark4man 0.400000 2022-02-17 00:00:00+00:00 True 4.136299 3.299236 -0.837063 -20.237010 3.103517 2.757547 ... -0.001771 -35.546726 13.576470 12.110069 -1.466401 -10.801048 620.444836 494.885355 -125.559481 -20.237010
../samples/2025_lw_what-s-the-short-timeline-plan.md What’s the short timeline plan? 0.898161 2025-01-05 00:10:28.708000+00:00 False 3.970715 3.319989 -0.650725 -16.388113 3.171675 2.998089 ... 0.000231 159.805322 13.405236 17.062244 3.657008 27.280446 623.402177 521.238323 -102.163854 -16.388113
../samples/2024_lw_the-plan-2024-update.md The Plan - 2024 Update 0.785170 2024-12-31 19:29:23.013000+00:00 False 3.741762 2.981632 -0.760130 -20.314768 2.872466 2.731928 ... 0.000043 70.526800 11.613052 11.740885 0.127832 1.100765 673.517207 536.693749 -136.823458 -20.314768
../samples/2025_lw_review-planecrash.md Review: Planecrash 0.933950 2025-01-01 01:31:26.864000+00:00 False 3.475632 1.752375 -1.723257 -49.581118 3.319073 2.316245 ... -0.000083 -79.884876 14.514912 10.882781 -3.632131 -25.023443 684.699408 345.217788 -339.481621 -49.581118
../samples/2025_lw_2024-in-ai-predictions.md 2024 in AI predictions 0.789190 2025-01-02 02:09:13.093000+00:00 False 4.635499 2.916743 -1.718757 -37.078135 3.080130 2.548945 ... 0.000238 50.615295 13.301698 12.898935 -0.402762 -3.027902 801.941370 504.596465 -297.344904 -37.078135

30 rows × 24 columns

In [88]:
#  df_res.sort_values('improvement%', ascending=False)
In [ ]:
print(df_res.to_markdown())

DEBUG

In [23]:
from IPython.display import display, HTML, Markdown
import torch

@torch.no_grad()
def gen(model, inputs, tokenizer, clean=True):
    s = model.generate(
        input_ids=inputs["input_ids"][None, :].to(model.device),
        attention_mask=inputs["attention_mask"][None, :].to(model.device),
        use_cache=False,
        max_new_tokens=100,
        min_new_tokens=100,
        do_sample=False,
        early_stopping=False,
    )
    input_l = inputs["input_ids"].shape[0]
    tokenizer_kwargs=dict(clean_up_tokenization_spaces=clean, skip_special_tokens=clean)
    old = tokenizer.decode(
        s[0, :input_l][-100:], **tokenizer_kwargs
    )
    new = tokenizer.decode(
        s[0, input_l:], **tokenizer_kwargs
    )
    s_old = ""+old.replace('\n', '<br>')
    s_new =  '<b>' + new.replace('\n', '<br>')+ '<br><br><b/>'
    # print(s_old, s_new)
    display(HTML(f"{s_old}{s_new}"))
    # print([old, new])
In [24]:
sample = samples[-1]
s = sample['content']
first_half = s[:len(s)//2]
second_half = s[len(s)//2:]
ds_train = Dataset.from_dict(tokenizer([first_half]))
ds_val = Dataset.from_dict(tokenizer([second_half]))
In [ ]:
with model.disable_adapter():
    gen(model, ds_train.with_format('pt')[0], tokenizer)
In [ ]:
gen(model, ds_train.with_format('pt')[0], tokenizer)
In [ ]:
In [ ]:
In [ ]:
Reference in New Issue View Git Blame Copy Permalink