detect_bs_text/nbs/01_detection_using_adapter_ft_split.ipynb

https://github.com/huggingface/peft/blob/main/examples/fp4_finetuning/finetune_fp4_opt_bnb_peft.py

In [1]:

from torch import optim
import lightning as pl
from matplotlib import pyplot as plt

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

In [2]:

from loguru import logger
import sys

# only if you want it shorter
logger.remove()
logger.add(sys.stderr, format="<level>{message}</level>", level="WARNING")

Out [2]:

1

In [3]:

import os
os.environ['CUDA_VISIBLE_DEVICES']="1"
# os.environ["CUDA_VISIBLE_DEVICES"] = "0"
import torch
import torch.nn as nn
import transformers
from datasets import load_dataset
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, AutoConfig
import numpy as np
from tqdm.auto import tqdm
import pandas as pd
import warnings
from peft import LoraConfig, get_peft_model, IA3Config

In [ ]:

In [4]:

plt.style.use('seaborn-v0_8')
torch.set_float32_matmul_precision('medium')
warnings.filterwarnings("ignore", ".*does not have many workers.*")
warnings.filterwarnings("ignore", ".*Was asked to gather along dimension 0.*")
warnings.filterwarnings("ignore", ".*There is an imbalance between your GPUs.*")

In [ ]:

In [5]:

# model_name = "TheBloke/phi-2-GPTQ"
model_name = "microsoft/phi-2"

# https://huggingface.co/collections/unsloth/llama-32-66f46afde4ca573864321a22
model_name = "unsloth/Llama-3.2-1B"
model_name = "unsloth/Llama-3.2-1B-bnb-4bit"
# Model Release Date: Sept 25, 2024
# launch date 9/25/2024 https://github.com/meta-llama/llama-models/blob/main/README.md
# https://colab.research.google.com/drive/1T5-zKWM_5OD21QHwXHiV9ixTRR7k3iB9?usp=sharing
# unsloth/Llama-3.2-3B
# Data Freshness: The pretraining data has a cutoff of December 2023.

def load_model():

    model = AutoModelForCausalLM.from_pretrained(
        model_name,
        # quantization_config=BitsAndBytesConfig(
        #     load_in_4bit=True,
        #     llm_int8_threshold=6.0,
        #     llm_int8_has_fp16_weight=False,
        #     bnb_4bit_compute_dtype=torch.float16,
        #     bnb_4bit_use_double_quant=True,
        #     bnb_4bit_quant_type="nf4",
        # ),
        torch_dtype=torch.float16,
        trust_remote_code=True,
    )


    # config = AutoConfig.from_pretrained(model_name, trust_remote_code=True,)
    # config.quantization_config['use_exllama'] = False
    # config.quantization_config['disable_exllama'] = True
    # model = AutoModelForCausalLM.from_pretrained(
    #     model_name,
    #     torch_dtype=torch.bfloat16,
    #     trust_remote_code=True,
    #     config=config,
    # )
    return model

In [6]:

base_model = load_model()
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True,)
tokenizer.pad_token = tokenizer.eos_token

Loading checkpoint shards: 100%|██████████| 2/2 [00:01<00:00,  1.77it/s]
Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.

In [7]:

def reset_model(base_model):
    # peft_config = LoraConfig(
    #     # task_type=TaskType.TOKEN_CLS, 
    #     target_modules=[ "fc2",  "Wqkv",],
    #     inference_mode=False, r=4, lora_alpha=4, 
    #     # lora_dropout=0.1, 
    #     # bias="all"
    # )
    # peft_config = IA3Config(
    #     target_modules=[ "fc2",  "Wqkv",], 
    #         feedforward_modules=["fc2"],
    #         inference_mode=False,
    # )
    peft_config = IA3Config(
        # target_modules=[ "fc2",  "Wqkv", 'out_proj', 'fc1'], 
        #     feedforward_modules=["fc2", 'fc1', 'out_proj'],
        #     inference_mode=False,
    )
    model = get_peft_model(base_model, peft_config)
    model.config.use_cache = False
    return model

model = reset_model(base_model)

In [8]:

MAX_LEN = 400

import frontmatter
from pathlib import Path
sample_files = sorted(Path("../samples/").glob('*.md'))
print(sample_files)
samples = [{'f':f, **frontmatter.load(f).to_dict()} for f in sample_files]

for sample in samples:
    assert 'title' in sample, sample['f']
    assert 'content' in sample
samples[0].keys()

Out [8]:

[PosixPath('../samples/bletchley_decleration.md'), PosixPath('../samples/cicero_fin1.md'), PosixPath('../samples/disney_appointment.md'), PosixPath('../samples/fake_paper.md'), PosixPath('../samples/fauci_emails.md'), PosixPath('../samples/harvard_announcement_reminders.md'), PosixPath('../samples/how_to_catch_a_liar.md'), PosixPath('../samples/lk-99_end.md'), PosixPath('../samples/lk-99_espanol.md'), PosixPath('../samples/lorem_ipsum.md'), PosixPath('../samples/openai_board_ann.md'), PosixPath('../samples/openai_paper_weak_to_strong.md'), PosixPath('../samples/politics_is_the_mind_killer.md'), PosixPath('../samples/statement_vyKamala_on_passing_of_johnson.md'), PosixPath('../samples/survey_of_rumours.md')]

dict_keys(['f', 'title', 'url', 'content'])

In [ ]:

Helpers

In [9]:

# modified from https://github.dev/huggingface/evaluate/blob/8dfe05784099fb9af55b8e77793205a3b7c86465/measurements/perplexity/perplexity.py#L154
import evaluate
from evaluate import logging
from torch.nn import CrossEntropyLoss
from torch.utils.data import DataLoader

def perplexity_compute(
    ds, model, tokenizer, batch_size: int = 16, add_start_token: bool = True, device=None, max_length=None
):
    model = model.to(device)


    ds = ds.with_format('pt')
    dl = DataLoader(ds, batch_size=1, shuffle=False, num_workers=0, collate_fn=tokenizer.pad, pin_memory=True)
    ppls = []
    loss_fct = CrossEntropyLoss(reduction="none")
    for b in dl:
        input_ids = b['input_ids'].to(device)
        attention_mask = b['attention_mask'].to(device)
        # print(attention_mask)

        labels = input_ids

        with torch.no_grad():
            out_logits = model(input_ids=input_ids, attention_mask=attention_mask).logits

        shift_logits = out_logits[..., :-1, :].contiguous()
        shift_labels = labels[..., 1:].contiguous()
        shift_attention_mask_batch = attention_mask[..., 1:].contiguous()

        perplexity_batch = torch.exp(
            (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch).sum(1)
            / shift_attention_mask_batch.sum(1)
        )

        ppls += perplexity_batch.tolist()

    return {"perplexities": ppls, "mean_perplexity": torch.tensor(ppls).mean()}

In [10]:

# perplexity_compute(ds=ds_val, model=model, tokenizer=tokenizer, device='cuda')

Training

In [11]:

from torch.nn import functional as F
from torch.utils.data import DataLoader, TensorDataset
from datasets import Dataset

Lightning helpers

In [12]:

def eval(model, tokenizer, ds_val: Dataset):
    model.eval();
    with torch.no_grad():
        with model.disable_adapter():
            results = perplexity_compute(ds=ds_val, model=model, tokenizer=tokenizer, device='cuda')
        results2 = perplexity_compute(ds=ds_val, model=model, tokenizer=tokenizer, device='cuda')
    return dict(before=results['mean_perplexity'].item(), after=results2['mean_perplexity'].item())

Train

In [13]:

from datasets import Dataset


def compute_metrics(eval_prediction):
    return {}

Trainer docs

• https://huggingface.co/docs/transformers/v4.36.1/en/main_classes/trainer#transformers.Trainer

In [14]:

MAX_LEN, len(sample['content'])//3

Out [14]:

(400, 14201)

In [15]:

from sklearn.model_selection import train_test_split

def tokenize_and_split(examples):
    l = len(tokenizer(examples).input_ids[0])
    max_len = min(l//3, MAX_LEN) # break into at least 5
    max_len = max(max_len, 10)


    result = tokenizer(
        examples,
        add_special_tokens=False,
        truncation=True,
        stride=2,
        max_length=max_len,
        return_overflowing_tokens=True,
        return_attention_mask=True,
    )
    return result

s = sample['content']
d = Dataset.from_dict(tokenize_and_split([s]))
d2  = d.train_test_split(test_size=0.5, seed=42)
ds_train = d2['train']
ds_val = d2['test']
ds_val

Out [15]:

Token indices sequence length is longer than the specified maximum sequence length for this model (8172 > 2048). Running this sequence through the model will result in indexing errors

Dataset({
    features: ['input_ids', 'attention_mask', 'overflow_to_sample_mapping'],
    num_rows: 11
})

In [ ]:

In [16]:

def learn_sample(sample):
    # device = 'cuda'
    # lr = 4e-3
    # epochs = 3
    # accum_steps = 1
    batch_size = 1
    verbose = False

    s = sample['content']

    d = Dataset.from_dict(tokenize_and_split([s]))
    d2  = d.train_test_split(test_size=0.5, seed=42)
    ds_train = d2['train']
    ds_val = d2['test']

    
    model = reset_model(base_model)
    # eval(model, tokenizer, ds_train)

    # https://huggingface.co/docs/transformers/v4.36.1/en/main_classes/trainer#transformers.Trainer
    trainer = transformers.Trainer(
        model=model,
        train_dataset=ds_train,
        eval_dataset=ds_val,
        compute_metrics=compute_metrics, # without this it wont even give val loss
        args=transformers.TrainingArguments(
            # checkpoint='epoch',
            save_strategy='epoch',
            label_names=['labels',],
            per_device_train_batch_size=batch_size,
            # gradient_accumulation_steps=1,
            # warmup_steps=6,
            warmup_ratio=0.1,
            # max_steps=50,
            num_train_epochs=3,
            learning_rate=1e-3,
            fp16=True,
            logging_steps=1,
            output_dir="outputs",
            log_level='error',
            # do_eval=True,
            evaluation_strategy="epoch",
            eval_steps=1,
            load_best_model_at_end=True,
            
            # disable_tqdm=not verbose,
        ),
        data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),
    )
    trainer._signature_columns = ['input_ids', 'attention_mask', 'labels',]
    model.config.use_cache = False  # silence the warnings. Please re-enable for inference!
    train_output = trainer.train()

    df_hist = pd.DataFrame(trainer.state.log_history)
    df_hist_epoch = df_hist.groupby('epoch').last().drop(columns=['step'])
    df_hist_step = df_hist.set_index('step').dropna(thresh=2, axis=1)
    if verbose:
        df_hist_epoch['loss'].plot()
        plt.twinx()
        df_hist_epoch['eval_loss'].plot(c='b', label='eval')
        plt.legend()
        plt.show()


    result_train = {f'train/{k}':v for k,v in eval(model, tokenizer, ds_train).items()}
    result = eval(model, tokenizer, ds_val)
    result['hist'] = df_hist_epoch
    result.update(result_train)
    return result

In [ ]:

In [17]:

data = []
for sample in samples:
    r = learn_sample(sample)
    print(sample['title'])
    print(dict(before=r['before'], after=r['after']))
    data.append(dict(**r, **sample))

 17%|█▋        | 1/6 [00:00<00:02,  2.35it/s]

{'loss': 2.8049, 'learning_rate': 0.001, 'epoch': 0.5}

 33%|███▎      | 2/6 [00:00<00:01,  2.80it/s]

{'loss': 2.3359, 'learning_rate': 0.0008, 'epoch': 1.0}

                                             
 33%|███▎      | 2/6 [00:00<00:01,  2.80it/s]

{'eval_loss': 2.8958373069763184, 'eval_runtime': 0.2605, 'eval_samples_per_second': 7.677, 'eval_steps_per_second': 3.839, 'epoch': 1.0}

 50%|█████     | 3/6 [00:01<00:01,  2.08it/s]

{'loss': 2.7644, 'learning_rate': 0.0006, 'epoch': 1.5}

 67%|██████▋   | 4/6 [00:01<00:00,  2.42it/s]

{'loss': 2.297, 'learning_rate': 0.0004, 'epoch': 2.0}

                                             
 67%|██████▋   | 4/6 [00:01<00:00,  2.42it/s]

{'eval_loss': 2.8877780437469482, 'eval_runtime': 0.2592, 'eval_samples_per_second': 7.715, 'eval_steps_per_second': 3.857, 'epoch': 2.0}

 83%|████████▎ | 5/6 [00:02<00:00,  2.05it/s]

{'loss': 2.2516, 'learning_rate': 0.0002, 'epoch': 2.5}

100%|██████████| 6/6 [00:02<00:00,  2.35it/s]

{'loss': 2.6997, 'learning_rate': 0.0, 'epoch': 3.0}

                                             
100%|██████████| 6/6 [00:02<00:00,  2.05it/s]

{'eval_loss': 2.884704113006592, 'eval_runtime': 0.2598, 'eval_samples_per_second': 7.699, 'eval_steps_per_second': 3.85, 'epoch': 3.0}
{'train_runtime': 2.9316, 'train_samples_per_second': 2.047, 'train_steps_per_second': 2.047, 'train_loss': 2.525577942530314, 'epoch': 3.0}
blechley declaration
{'before': 17.869091033935547, 'after': 17.52420425415039}
{'loss': 4.625, 'learning_rate': 0.0005, 'epoch': 0.17}
{'loss': 4.5422, 'learning_rate': 0.001, 'epoch': 0.33}
{'loss': 4.5928, 'learning_rate': 0.0009375, 'epoch': 0.5}
{'loss': 4.5384, 'learning_rate': 0.000875, 'epoch': 0.67}
{'loss': 4.7047, 'learning_rate': 0.0008125000000000001, 'epoch': 0.83}
{'loss': 4.4394, 'learning_rate': 0.00075, 'epoch': 1.0}
{'eval_loss': 4.234132289886475, 'eval_runtime': 0.8515, 'eval_samples_per_second': 7.047, 'eval_steps_per_second': 1.174, 'epoch': 1.0}
{'loss': 4.4981, 'learning_rate': 0.0006875, 'epoch': 1.17}
{'loss': 4.5187, 'learning_rate': 0.000625, 'epoch': 1.33}
{'loss': 4.419, 'learning_rate': 0.0005625000000000001, 'epoch': 1.5}
{'loss': 4.6345, 'learning_rate': 0.0005, 'epoch': 1.67}
{'loss': 4.4387, 'learning_rate': 0.0004375, 'epoch': 1.83}
{'loss': 4.3689, 'learning_rate': 0.000375, 'epoch': 2.0}
{'eval_loss': 4.212051868438721, 'eval_runtime': 0.7417, 'eval_samples_per_second': 8.09, 'eval_steps_per_second': 1.348, 'epoch': 2.0}
{'loss': 4.3709, 'learning_rate': 0.0003125, 'epoch': 2.17}
{'loss': 4.4461, 'learning_rate': 0.00025, 'epoch': 2.33}
{'loss': 4.5014, 'learning_rate': 0.0001875, 'epoch': 2.5}
{'loss': 4.5928, 'learning_rate': 0.000125, 'epoch': 2.67}
{'loss': 4.4311, 'learning_rate': 6.25e-05, 'epoch': 2.83}
{'loss': 4.4222, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 4.2056355476379395, 'eval_runtime': 0.7622, 'eval_samples_per_second': 7.872, 'eval_steps_per_second': 1.312, 'epoch': 3.0}
{'train_runtime': 8.1788, 'train_samples_per_second': 2.201, 'train_steps_per_second': 2.201, 'train_loss': 4.504713243908352, 'epoch': 3.0}
cicero from ibois, Philippe (2012-06-03).
{'before': 72.58737182617188, 'after': 67.74419403076172}
{'loss': 2.187, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 2.0009, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 1.9654667377471924, 'eval_runtime': 0.178, 'eval_samples_per_second': 11.238, 'eval_steps_per_second': 5.619, 'epoch': 1.0}
{'loss': 2.1424, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 1.9167, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 1.9636929035186768, 'eval_runtime': 0.1788, 'eval_samples_per_second': 11.188, 'eval_steps_per_second': 5.594, 'epoch': 2.0}
{'loss': 1.8877, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.0672, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 1.9641517400741577, 'eval_runtime': 0.184, 'eval_samples_per_second': 10.869, 'eval_steps_per_second': 5.435, 'epoch': 3.0}
{'train_runtime': 1.978, 'train_samples_per_second': 3.033, 'train_steps_per_second': 3.033, 'train_loss': 2.033644914627075, 'epoch': 3.0}
disney appointment
{'before': 118.82615661621094, 'after': 118.42012786865234}
{'loss': 2.0357, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 1.9873, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 1.807045817375183, 'eval_runtime': 0.1406, 'eval_samples_per_second': 14.221, 'eval_steps_per_second': 7.11, 'epoch': 1.0}
{'loss': 2.0176, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 1.9243, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 1.7954585552215576, 'eval_runtime': 0.1396, 'eval_samples_per_second': 14.328, 'eval_steps_per_second': 7.164, 'epoch': 2.0}
{'loss': 1.9109, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 1.9423, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 1.7919256687164307, 'eval_runtime': 0.14, 'eval_samples_per_second': 14.289, 'eval_steps_per_second': 7.145, 'epoch': 3.0}
{'train_runtime': 1.5666, 'train_samples_per_second': 3.83, 'train_steps_per_second': 3.83, 'train_loss': 1.969655990600586, 'epoch': 3.0}
fake ai hoax paper
{'before': 7.766979217529297, 'after': 7.697228908538818}
{'loss': 2.4331, 'learning_rate': 0.0005, 'epoch': 0.2}
{'loss': 2.9004, 'learning_rate': 0.001, 'epoch': 0.4}
{'loss': 3.5675, 'learning_rate': 0.0009230769230769232, 'epoch': 0.6}
{'loss': 3.3972, 'learning_rate': 0.0008461538461538462, 'epoch': 0.8}
{'loss': 3.5431, 'learning_rate': 0.0007692307692307693, 'epoch': 1.0}
{'eval_loss': 3.0353567600250244, 'eval_runtime': 0.8628, 'eval_samples_per_second': 6.954, 'eval_steps_per_second': 1.159, 'epoch': 1.0}
{'loss': 3.4732, 'learning_rate': 0.0006923076923076923, 'epoch': 1.2}
{'loss': 3.5319, 'learning_rate': 0.0006153846153846154, 'epoch': 1.4}
{'loss': 2.7887, 'learning_rate': 0.0005384615384615384, 'epoch': 1.6}
{'loss': 2.3222, 'learning_rate': 0.0004615384615384616, 'epoch': 1.8}
{'loss': 3.3527, 'learning_rate': 0.00038461538461538467, 'epoch': 2.0}
{'eval_loss': 3.0194509029388428, 'eval_runtime': 0.7424, 'eval_samples_per_second': 8.082, 'eval_steps_per_second': 1.347, 'epoch': 2.0}
{'loss': 2.2585, 'learning_rate': 0.0003076923076923077, 'epoch': 2.2}
{'loss': 3.3084, 'learning_rate': 0.0002307692307692308, 'epoch': 2.4}
{'loss': 3.4762, 'learning_rate': 0.00015384615384615385, 'epoch': 2.6}
{'loss': 3.2907, 'learning_rate': 7.692307692307693e-05, 'epoch': 2.8}
{'loss': 2.7338, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.014026641845703, 'eval_runtime': 0.7435, 'eval_samples_per_second': 8.07, 'eval_steps_per_second': 1.345, 'epoch': 3.0}
{'train_runtime': 7.2582, 'train_samples_per_second': 2.067, 'train_steps_per_second': 2.067, 'train_loss': 3.09183349609375, 'epoch': 3.0}
buzzfeed foi fauci emails 2023
{'before': 23.3094425201416, 'after': 22.406410217285156}
{'loss': 2.6779, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 2.4325, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 2.7339022159576416, 'eval_runtime': 0.2193, 'eval_samples_per_second': 9.121, 'eval_steps_per_second': 4.561, 'epoch': 1.0}
{'loss': 2.6596, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.3446, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 2.730224609375, 'eval_runtime': 0.2245, 'eval_samples_per_second': 8.91, 'eval_steps_per_second': 4.455, 'epoch': 2.0}
{'loss': 2.3111, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.5791, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 2.729649543762207, 'eval_runtime': 0.2204, 'eval_samples_per_second': 9.076, 'eval_steps_per_second': 4.538, 'epoch': 3.0}
{'train_runtime': 2.2643, 'train_samples_per_second': 2.65, 'train_steps_per_second': 2.65, 'train_loss': 2.500802000363668, 'epoch': 3.0}
harvard announcment caplain israel hamas
{'before': 45.347354888916016, 'after': 45.12727355957031}
{'loss': 2.6223, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 2.4621, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 3.127470016479492, 'eval_runtime': 0.378, 'eval_samples_per_second': 5.291, 'eval_steps_per_second': 2.645, 'epoch': 1.0}
{'loss': 2.5923, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.4313, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 3.115180253982544, 'eval_runtime': 0.2642, 'eval_samples_per_second': 7.569, 'eval_steps_per_second': 3.785, 'epoch': 2.0}
{'loss': 2.3652, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.5058, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 3.1118650436401367, 'eval_runtime': 0.2632, 'eval_samples_per_second': 7.6, 'eval_steps_per_second': 3.8, 'epoch': 3.0}
{'train_runtime': 2.9644, 'train_samples_per_second': 2.024, 'train_steps_per_second': 2.024, 'train_loss': 2.4964850743611655, 'epoch': 3.0}
How to Catch an AI Liar
{'before': 28.949859619140625, 'after': 28.0087890625}
{'loss': 3.5646, 'learning_rate': 0.0005, 'epoch': 0.2}
{'loss': 2.8252, 'learning_rate': 0.001, 'epoch': 0.4}
{'loss': 2.0116, 'learning_rate': 0.0009230769230769232, 'epoch': 0.6}
{'loss': 2.7551, 'learning_rate': 0.0008461538461538462, 'epoch': 0.8}
{'loss': 2.4442, 'learning_rate': 0.0007692307692307693, 'epoch': 1.0}
{'eval_loss': 2.635437488555908, 'eval_runtime': 0.7632, 'eval_samples_per_second': 6.552, 'eval_steps_per_second': 1.31, 'epoch': 1.0}
{'loss': 1.9103, 'learning_rate': 0.0006923076923076923, 'epoch': 1.2}
{'loss': 2.4336, 'learning_rate': 0.0006153846153846154, 'epoch': 1.4}
{'loss': 2.7349, 'learning_rate': 0.0005384615384615384, 'epoch': 1.6}
{'loss': 3.4216, 'learning_rate': 0.0004615384615384616, 'epoch': 1.8}
{'loss': 2.6993, 'learning_rate': 0.00038461538461538467, 'epoch': 2.0}
{'eval_loss': 2.623994827270508, 'eval_runtime': 0.6368, 'eval_samples_per_second': 7.852, 'eval_steps_per_second': 1.57, 'epoch': 2.0}
{'loss': 3.3393, 'learning_rate': 0.0003076923076923077, 'epoch': 2.2}
{'loss': 1.8571, 'learning_rate': 0.0002307692307692308, 'epoch': 2.4}
{'loss': 2.3943, 'learning_rate': 0.00015384615384615385, 'epoch': 2.6}
{'loss': 2.6463, 'learning_rate': 7.692307692307693e-05, 'epoch': 2.8}
{'loss': 2.6925, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 2.6198740005493164, 'eval_runtime': 0.636, 'eval_samples_per_second': 7.862, 'eval_steps_per_second': 1.572, 'epoch': 3.0}
{'train_runtime': 6.4124, 'train_samples_per_second': 2.339, 'train_steps_per_second': 2.339, 'train_loss': 2.648661724726359, 'epoch': 3.0}
LK-99-en
{'before': 14.513845443725586, 'after': 14.066093444824219}
{'loss': 2.9979, 'learning_rate': 0.0005, 'epoch': 0.17}
{'loss': 2.0904, 'learning_rate': 0.001, 'epoch': 0.33}
{'loss': 2.7862, 'learning_rate': 0.0009375, 'epoch': 0.5}
{'loss': 2.4246, 'learning_rate': 0.000875, 'epoch': 0.67}
{'loss': 2.6095, 'learning_rate': 0.0008125000000000001, 'epoch': 0.83}
{'loss': 2.5655, 'learning_rate': 0.00075, 'epoch': 1.0}
{'eval_loss': 2.335665464401245, 'eval_runtime': 0.9768, 'eval_samples_per_second': 7.166, 'eval_steps_per_second': 1.024, 'epoch': 1.0}
{'loss': 2.3562, 'learning_rate': 0.0006875, 'epoch': 1.17}
{'loss': 2.8196, 'learning_rate': 0.000625, 'epoch': 1.33}
{'loss': 2.5404, 'learning_rate': 0.0005625000000000001, 'epoch': 1.5}
{'loss': 2.4861, 'learning_rate': 0.0005, 'epoch': 1.67}
{'loss': 2.6823, 'learning_rate': 0.0004375, 'epoch': 1.83}
{'loss': 2.0126, 'learning_rate': 0.000375, 'epoch': 2.0}
{'eval_loss': 2.320927858352661, 'eval_runtime': 0.867, 'eval_samples_per_second': 8.074, 'eval_steps_per_second': 1.153, 'epoch': 2.0}
{'loss': 2.4835, 'learning_rate': 0.0003125, 'epoch': 2.17}
{'loss': 2.5887, 'learning_rate': 0.00025, 'epoch': 2.33}
{'loss': 2.7585, 'learning_rate': 0.0001875, 'epoch': 2.5}
{'loss': 2.4708, 'learning_rate': 0.000125, 'epoch': 2.67}
{'loss': 2.2034, 'learning_rate': 6.25e-05, 'epoch': 2.83}
{'loss': 1.9709, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 2.3152239322662354, 'eval_runtime': 0.8652, 'eval_samples_per_second': 8.091, 'eval_steps_per_second': 1.156, 'epoch': 3.0}
{'train_runtime': 8.6099, 'train_samples_per_second': 2.091, 'train_steps_per_second': 2.091, 'train_loss': 2.4914957218699985, 'epoch': 3.0}
LK-99-es
{'before': 11.415018081665039, 'after': 10.97294807434082}
{'loss': 1.8845, 'learning_rate': 0.0003333333333333333, 'epoch': 0.11}
{'loss': 1.9495, 'learning_rate': 0.0006666666666666666, 'epoch': 0.22}
{'loss': 1.9686, 'learning_rate': 0.001, 'epoch': 0.33}
{'loss': 1.9168, 'learning_rate': 0.0009583333333333334, 'epoch': 0.44}
{'loss': 1.9844, 'learning_rate': 0.0009166666666666666, 'epoch': 0.56}
{'loss': 1.9139, 'learning_rate': 0.000875, 'epoch': 0.67}
{'loss': 1.8329, 'learning_rate': 0.0008333333333333334, 'epoch': 0.78}
{'loss': 1.8556, 'learning_rate': 0.0007916666666666666, 'epoch': 0.89}
{'loss': 1.8869, 'learning_rate': 0.0007916666666666666, 'epoch': 1.0}
{'eval_loss': 1.836249828338623, 'eval_runtime': 1.2543, 'eval_samples_per_second': 7.175, 'eval_steps_per_second': 1.595, 'epoch': 1.0}
{'loss': 1.8902, 'learning_rate': 0.00075, 'epoch': 1.11}
{'loss': 1.8931, 'learning_rate': 0.0007083333333333334, 'epoch': 1.22}
{'loss': 1.8021, 'learning_rate': 0.0006666666666666666, 'epoch': 1.33}
{'loss': 1.884, 'learning_rate': 0.000625, 'epoch': 1.44}
{'loss': 1.7923, 'learning_rate': 0.0005833333333333334, 'epoch': 1.56}
{'loss': 1.848, 'learning_rate': 0.0005416666666666666, 'epoch': 1.67}
{'loss': 1.8103, 'learning_rate': 0.0005, 'epoch': 1.78}
{'loss': 1.9024, 'learning_rate': 0.0004583333333333333, 'epoch': 1.89}
{'loss': 1.9309, 'learning_rate': 0.0004166666666666667, 'epoch': 2.0}
{'eval_loss': 1.829906702041626, 'eval_runtime': 1.1467, 'eval_samples_per_second': 7.849, 'eval_steps_per_second': 1.744, 'epoch': 2.0}
{'loss': 1.9011, 'learning_rate': 0.000375, 'epoch': 2.11}
{'loss': 1.8506, 'learning_rate': 0.0003333333333333333, 'epoch': 2.22}
{'loss': 1.8036, 'learning_rate': 0.0002916666666666667, 'epoch': 2.33}
{'loss': 1.828, 'learning_rate': 0.00025, 'epoch': 2.44}
{'loss': 1.9094, 'learning_rate': 0.00020833333333333335, 'epoch': 2.56}
{'loss': 1.8678, 'learning_rate': 0.00016666666666666666, 'epoch': 2.67}
{'loss': 1.8555, 'learning_rate': 0.000125, 'epoch': 2.78}
{'loss': 1.7289, 'learning_rate': 8.333333333333333e-05, 'epoch': 2.89}
{'loss': 1.7859, 'learning_rate': 4.1666666666666665e-05, 'epoch': 3.0}
{'eval_loss': 1.8297154903411865, 'eval_runtime': 1.1477, 'eval_samples_per_second': 7.841, 'eval_steps_per_second': 1.743, 'epoch': 3.0}
{'train_runtime': 12.5295, 'train_samples_per_second': 2.155, 'train_steps_per_second': 2.155, 'train_loss': 1.8695231985162806, 'epoch': 3.0}
Lorem ipsum
{'before': 6.564840316772461, 'after': 6.268791675567627}
{'loss': 2.2808, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 2.264, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 2.300767183303833, 'eval_runtime': 0.1412, 'eval_samples_per_second': 14.167, 'eval_steps_per_second': 7.084, 'epoch': 1.0}
{'loss': 2.1983, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.1913, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 2.2839250564575195, 'eval_runtime': 0.1403, 'eval_samples_per_second': 14.254, 'eval_steps_per_second': 7.127, 'epoch': 2.0}
{'loss': 2.149, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.095, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 2.2791788578033447, 'eval_runtime': 0.1407, 'eval_samples_per_second': 14.211, 'eval_steps_per_second': 7.105, 'epoch': 3.0}
{'train_runtime': 1.5544, 'train_samples_per_second': 3.86, 'train_steps_per_second': 3.86, 'train_loss': 2.196397304534912, 'epoch': 3.0}
openai board ann
{'before': 55.808502197265625, 'after': 54.66792297363281}
{'loss': 2.8464, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 2.4796, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 2.501676082611084, 'eval_runtime': 0.3861, 'eval_samples_per_second': 5.18, 'eval_steps_per_second': 2.59, 'epoch': 1.0}
{'loss': 2.8062, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.4146, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 2.4917240142822266, 'eval_runtime': 0.2752, 'eval_samples_per_second': 7.268, 'eval_steps_per_second': 3.634, 'epoch': 2.0}
{'loss': 2.356, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.7228, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 2.488405704498291, 'eval_runtime': 0.2661, 'eval_samples_per_second': 7.515, 'eval_steps_per_second': 3.757, 'epoch': 3.0}
{'train_runtime': 3.0083, 'train_samples_per_second': 1.994, 'train_steps_per_second': 1.994, 'train_loss': 2.60427188873291, 'epoch': 3.0}
weak to strong
{'before': 46.764183044433594, 'after': 46.40473556518555}
{'loss': 2.7259, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 2.7937, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 2.7441396713256836, 'eval_runtime': 0.1515, 'eval_samples_per_second': 13.205, 'eval_steps_per_second': 6.603, 'epoch': 1.0}
{'loss': 2.6661, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.6998, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 2.74137282371521, 'eval_runtime': 0.1514, 'eval_samples_per_second': 13.211, 'eval_steps_per_second': 6.605, 'epoch': 2.0}
{'loss': 2.6949, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 2.5599, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 2.7393088340759277, 'eval_runtime': 0.1558, 'eval_samples_per_second': 12.833, 'eval_steps_per_second': 6.416, 'epoch': 3.0}
{'train_runtime': 1.6635, 'train_samples_per_second': 3.607, 'train_steps_per_second': 3.607, 'train_loss': 2.690062483151754, 'epoch': 3.0}
politics is the mind-killer
{'before': 247.5515594482422, 'after': 245.82681274414062}
{'loss': 1.8812, 'learning_rate': 0.001, 'epoch': 0.5}
{'loss': 2.2442, 'learning_rate': 0.0008, 'epoch': 1.0}
{'eval_loss': 2.4278409481048584, 'eval_runtime': 0.0735, 'eval_samples_per_second': 27.196, 'eval_steps_per_second': 13.598, 'epoch': 1.0}
{'loss': 1.8079, 'learning_rate': 0.0006, 'epoch': 1.5}
{'loss': 2.1305, 'learning_rate': 0.0004, 'epoch': 2.0}
{'eval_loss': 2.4188129901885986, 'eval_runtime': 0.0764, 'eval_samples_per_second': 26.194, 'eval_steps_per_second': 13.097, 'epoch': 2.0}
{'loss': 2.1037, 'learning_rate': 0.0002, 'epoch': 2.5}
{'loss': 1.6808, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 2.4169728755950928, 'eval_runtime': 0.0738, 'eval_samples_per_second': 27.114, 'eval_steps_per_second': 13.557, 'epoch': 3.0}
{'train_runtime': 0.9463, 'train_samples_per_second': 6.341, 'train_steps_per_second': 6.341, 'train_loss': 1.9747143586476643, 'epoch': 3.0}
statement by whitehouse on passing
{'before': 29.197132110595703, 'after': 28.9396915435791}
{'loss': 2.5398, 'learning_rate': 0.0003333333333333333, 'epoch': 0.1}
{'loss': 2.563, 'learning_rate': 0.0006666666666666666, 'epoch': 0.2}
{'loss': 2.362, 'learning_rate': 0.001, 'epoch': 0.3}
{'loss': 2.4908, 'learning_rate': 0.0009629629629629629, 'epoch': 0.4}
{'loss': 2.8433, 'learning_rate': 0.000925925925925926, 'epoch': 0.5}
{'loss': 3.1541, 'learning_rate': 0.0008888888888888888, 'epoch': 0.6}
{'loss': 2.9383, 'learning_rate': 0.0008518518518518519, 'epoch': 0.7}
{'loss': 2.6576, 'learning_rate': 0.0008148148148148148, 'epoch': 0.8}
{'loss': 2.6327, 'learning_rate': 0.0007777777777777778, 'epoch': 0.9}
{'loss': 2.6943, 'learning_rate': 0.0007407407407407407, 'epoch': 1.0}
{'eval_loss': 2.42354416847229, 'eval_runtime': 1.5423, 'eval_samples_per_second': 7.132, 'eval_steps_per_second': 1.297, 'epoch': 1.0}
{'loss': 2.3882, 'learning_rate': 0.0007037037037037038, 'epoch': 1.1}
{'loss': 2.738, 'learning_rate': 0.0006666666666666666, 'epoch': 1.2}
{'loss': 2.5917, 'learning_rate': 0.0006296296296296296, 'epoch': 1.3}
{'loss': 2.8582, 'learning_rate': 0.0005925925925925926, 'epoch': 1.4}
{'loss': 3.0774, 'learning_rate': 0.0005555555555555556, 'epoch': 1.5}
{'loss': 2.266, 'learning_rate': 0.0005185185185185185, 'epoch': 1.6}
{'loss': 2.4382, 'learning_rate': 0.00048148148148148144, 'epoch': 1.7}
{'loss': 2.6197, 'learning_rate': 0.0004444444444444444, 'epoch': 1.8}
{'loss': 2.3982, 'learning_rate': 0.0004074074074074074, 'epoch': 1.9}
{'loss': 2.5713, 'learning_rate': 0.00037037037037037035, 'epoch': 2.0}
{'eval_loss': 2.4057815074920654, 'eval_runtime': 1.6782, 'eval_samples_per_second': 6.555, 'eval_steps_per_second': 1.192, 'epoch': 2.0}
{'loss': 2.6508, 'learning_rate': 0.0003333333333333333, 'epoch': 2.1}
{'loss': 2.4209, 'learning_rate': 0.0002962962962962963, 'epoch': 2.2}
{'loss': 3.0109, 'learning_rate': 0.00025925925925925926, 'epoch': 2.3}
{'loss': 2.561, 'learning_rate': 0.0002222222222222222, 'epoch': 2.4}
{'loss': 2.6675, 'learning_rate': 0.00018518518518518518, 'epoch': 2.5}
{'loss': 2.4846, 'learning_rate': 0.00014814814814814815, 'epoch': 2.6}
{'loss': 2.7888, 'learning_rate': 0.0001111111111111111, 'epoch': 2.7}
{'loss': 2.327, 'learning_rate': 7.407407407407407e-05, 'epoch': 2.8}
{'loss': 2.3715, 'learning_rate': 3.7037037037037037e-05, 'epoch': 2.9}
{'loss': 2.2409, 'learning_rate': 0.0, 'epoch': 3.0}
{'eval_loss': 2.399984121322632, 'eval_runtime': 1.608, 'eval_samples_per_second': 6.841, 'eval_steps_per_second': 1.244, 'epoch': 3.0}
{'train_runtime': 15.0837, 'train_samples_per_second': 1.989, 'train_steps_per_second': 1.989, 'train_loss': 2.6115529775619506, 'epoch': 3.0}
Gemini to Q*
{'before': 11.756383895874023, 'after': 11.190644264221191}

In [18]:

# example training
df_hist = data[-1]['hist']#.groupby('epoch').last().dropna(axis=1).drop(columns=['step'])
df_hist['loss'].plot(label='train')
plt.twinx()
df_hist['eval_loss'].plot(c='b', label='eval')
plt.legend()
plt.show()

In [19]:

df_hist['learning_rate'].plot(logy=True)

Out [19]:

<Axes: xlabel='epoch'>

Perplexity

Perplexity measures how well a language model predicts a text sample. Lower is better

It’s calculated as the average number of bits per word a model needs to represent the same

https://huggingface.co/docs/transformers/perplexity https://thegradient.pub/understanding-evaluation-metrics-for-language-models/

The improvement column, is perplexity decrease

In [ ]:

In [28]:

df_res = pd.DataFrame(data)
df_res['len'] = df_res.content.str.len()
df_res = df_res[['before', 'after', 'title',  'len']].set_index('title')
df_res['improvement%'] = (df_res['before'] - df_res['after'])/ df_res['before']
df_res['improvement'] = (df_res['before'] - df_res['after'])
df_res['novel'] = df_res['before'] > 15
df_res['learnable'] = df_res['improvement%'] > 0.02

# We can measure the final score using learnable * novel
# df_res['BS'] = ~df_res['learnable'] | ~df_res['novel']
# Or just absolute perplexity improvement
df_res['BS'] = df_res['improvement'] < .3
df_res = df_res.sort_values('improvement', ascending=False)
df_res

Out [28]:

	before	after	len	improvement%	improvement	novel	learnable	BS
title
cicero from ibois, Philippe (2012-06-03).	72.587372	67.744194	13707	0.066722	4.843178	True	True	False
politics is the mind-killer	247.551559	245.826813	3158	0.006967	1.724747	True	False	False
openai board ann	55.808502	54.667923	2991	0.020437	1.140579	True	True	False
How to Catch an AI Liar	28.949860	28.008789	5464	0.032507	0.941071	True	True	False
buzzfeed foi fauci emails 2023	23.309443	22.406410	13640	0.038741	0.903032	True	True	False
Gemini to Q*	11.756384	11.190644	42604	0.048122	0.565740	False	True	False
LK-99-en	14.513845	14.066093	15432	0.030850	0.447752	False	True	False
LK-99-es	11.415018	10.972948	12970	0.038727	0.442070	False	True	False
disney appointment	118.826157	118.420128	3653	0.003417	0.406029	True	False	False
weak to strong	46.764183	46.404736	5811	0.007686	0.359447	True	False	False
blechley declaration	17.869091	17.524204	7762	0.019301	0.344887	True	False	False
Lorem ipsum	6.564840	6.268792	19649	0.045096	0.296049	False	True	True
statement by whitehouse on passing	29.197132	28.939692	1641	0.008817	0.257441	True	False	True
harvard announcment caplain israel hamas	45.347355	45.127274	4247	0.004853	0.220081	True	False	True
fake ai hoax paper	7.766979	7.697229	3290	0.008980	0.069750	False	False	True

In [26]:

#  df_res.sort_values('improvement%', ascending=False)

In [22]:

print(df_res.to_markdown())

| title                                     |    before |     after |   len |   improvement% |   improvement | novel   | learnable   | BS    |
|:------------------------------------------|----------:|----------:|------:|---------------:|--------------:|:--------|:------------|:------|
| cicero from ibois, Philippe (2012-06-03). |  72.5874  |  67.7442  | 13707 |     0.066722   |     4.84318   | True    | True        | False |
| politics is the mind-killer               | 247.552   | 245.827   |  3158 |     0.00696722 |     1.72475   | True    | False       | False |
| openai board ann                          |  55.8085  |  54.6679  |  2991 |     0.0204374  |     1.14058   | True    | True        | False |
| How to Catch an AI Liar                   |  28.9499  |  28.0088  |  5464 |     0.0325069  |     0.941071  | True    | True        | True  |
| buzzfeed foi fauci emails 2023            |  23.3094  |  22.4064  | 13640 |     0.0387411  |     0.903032  | True    | True        | True  |
| Gemini to Q*                              |  11.7564  |  11.1906  | 42604 |     0.0481219  |     0.56574   | False   | True        | True  |
| LK-99-en                                  |  14.5138  |  14.0661  | 15432 |     0.03085    |     0.447752  | False   | True        | True  |
| LK-99-es                                  |  11.415   |  10.9729  | 12970 |     0.0387271  |     0.44207   | False   | True        | True  |
| disney appointment                        | 118.826   | 118.42    |  3653 |     0.003417   |     0.406029  | True    | False       | True  |
| weak to strong                            |  46.7642  |  46.4047  |  5811 |     0.00768638 |     0.359447  | True    | False       | True  |
| blechley declaration                      |  17.8691  |  17.5242  |  7762 |     0.0193007  |     0.344887  | True    | False       | True  |
| Lorem ipsum                               |   6.56484 |   6.26879 | 19649 |     0.0450961  |     0.296049  | False   | True        | True  |
| statement by whitehouse on passing        |  29.1971  |  28.9397  |  1641 |     0.00881732 |     0.257441  | True    | False       | True  |
| harvard announcment caplain israel hamas  |  45.3474  |  45.1273  |  4247 |     0.00485323 |     0.220081  | True    | False       | True  |
| fake ai hoax paper                        |   7.76698 |   7.69723 |  3290 |     0.00898037 |     0.0697503 | False   | False       | True  |

DEBUG

In [23]:

from IPython.display import display, HTML, Markdown
import torch

@torch.no_grad()
def gen(model, inputs, tokenizer, clean=True):
    s = model.generate(
        input_ids=inputs["input_ids"][None, :].to(model.device),
        attention_mask=inputs["attention_mask"][None, :].to(model.device),
        use_cache=False,
        max_new_tokens=100,
        min_new_tokens=100,
        do_sample=False,
        early_stopping=False,
    )
    input_l = inputs["input_ids"].shape[0]
    tokenizer_kwargs=dict(clean_up_tokenization_spaces=clean, skip_special_tokens=clean)
    old = tokenizer.decode(
        s[0, :input_l][-100:], **tokenizer_kwargs
    )
    new = tokenizer.decode(
        s[0, input_l:], **tokenizer_kwargs
    )
    s_old = ""+old.replace('\n', '<br>')
    s_new =  '<b>' + new.replace('\n', '<br>')+ '<br><br><b/>'
    # print(s_old, s_new)
    display(HTML(f"{s_old}{s_new}"))
    # print([old, new])

In [24]:

sample = samples[-1]
s = sample['content']
first_half = s[:len(s)//2]
second_half = s[len(s)//2:]
ds_train = Dataset.from_dict(tokenizer([first_half]))
ds_val = Dataset.from_dict(tokenizer([second_half]))

In [25]:

with model.disable_adapter():
    gen(model, ds_train.with_format('pt')[0], tokenizer)

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/generation/utils.py:1421: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use and modify the model generation configuration (see https://huggingface.co/docs/transformers/generation_strategies#default-text-generation-configuration )
  warnings.warn(

---------------------------------------------------------------------------
OutOfMemoryError                          Traceback (most recent call last)
Cell In[25], line 2
      1 with model.disable_adapter():
----> 2     gen(model, ds_train.with_format('pt')[0], tokenizer)

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/utils/_contextlib.py:115, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    112 @functools.wraps(func)
    113 def decorate_context(*args, **kwargs):
    114     with ctx_factory():
--> 115         return func(*args, **kwargs)

Cell In[23], line 6, in gen(model, inputs, tokenizer, clean)
      4 @torch.no_grad()
      5 def gen(model, inputs, tokenizer, clean=True):
----> 6     s = model.generate(
      7         input_ids=inputs["input_ids"][None, :].to(model.device),
      8         attention_mask=inputs["attention_mask"][None, :].to(model.device),
      9         use_cache=False,
     10         max_new_tokens=100,
     11         min_new_tokens=100,
     12         do_sample=False,
     13         early_stopping=False,
     14     )
     15     input_l = inputs["input_ids"].shape[0]
     16     tokenizer_kwargs=dict(clean_up_tokenization_spaces=clean, skip_special_tokens=clean)

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/utils/_contextlib.py:115, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    112 @functools.wraps(func)
    113 def decorate_context(*args, **kwargs):
    114     with ctx_factory():
--> 115         return func(*args, **kwargs)

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/generation/utils.py:1606, in GenerationMixin.generate(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, negative_prompt_ids, negative_prompt_attention_mask, **kwargs)
   1589     return self.assisted_decoding(
   1590         input_ids,
   1591         assistant_model=assistant_model,
   (...)
   1602         **model_kwargs,
   1603     )
   1604 if generation_mode == GenerationMode.GREEDY_SEARCH:
   1605     # 11. run greedy search
-> 1606     return self.greedy_search(
   1607         input_ids,
   1608         logits_processor=logits_processor,
   1609         stopping_criteria=stopping_criteria,
   1610         pad_token_id=generation_config.pad_token_id,
   1611         eos_token_id=generation_config.eos_token_id,
   1612         output_scores=generation_config.output_scores,
   1613         return_dict_in_generate=generation_config.return_dict_in_generate,
   1614         synced_gpus=synced_gpus,
   1615         streamer=streamer,
   1616         **model_kwargs,
   1617     )
   1619 elif generation_mode == GenerationMode.CONTRASTIVE_SEARCH:
   1620     if not model_kwargs["use_cache"]:

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/generation/utils.py:2454, in GenerationMixin.greedy_search(self, input_ids, logits_processor, stopping_criteria, max_length, pad_token_id, eos_token_id, output_attentions, output_hidden_states, output_scores, return_dict_in_generate, synced_gpus, streamer, **model_kwargs)
   2451 model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
   2453 # forward pass to get next token
-> 2454 outputs = self(
   2455     **model_inputs,
   2456     return_dict=True,
   2457     output_attentions=output_attentions,
   2458     output_hidden_states=output_hidden_states,
   2459 )
   2461 if synced_gpus and this_peer_finished:
   2462     continue  # don't waste resources running the code we don't need

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File ~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:953, in PhiForCausalLM.forward(self, input_ids, past_key_values, attention_mask, labels, **kwargs)
    945 def forward(
    946     self,
    947     input_ids: torch.LongTensor,
   (...)
    951     **kwargs,
    952 ) -> CausalLMOutputWithPast:
--> 953     hidden_states = self.transformer(input_ids, past_key_values=past_key_values, attention_mask=attention_mask)
    954     lm_logits = self.lm_head(hidden_states)
    956     loss = None

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File ~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:915, in PhiModel.forward(self, input_ids, past_key_values, attention_mask)
    912 hidden_states = self.embd(input_ids)
    914 for layer in self.h:
--> 915     hidden_states = layer(
    916         hidden_states,
    917         past_key_values=past_key_values,
    918         attention_mask=attention_mask,
    919     )
    921 return hidden_states

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File ~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:770, in ParallelBlock.forward(self, hidden_states, past_key_values, attention_mask, **kwargs)
    767 residual = hidden_states
    768 hidden_states = self.ln(hidden_states)
--> 770 attn_outputs = self.mixer(
    771     hidden_states,
    772     past_key_values=past_key_values,
    773     attention_mask=attention_mask,
    774 )
    775 if isinstance(attn_outputs, tuple):
    776     attn_outputs = attn_outputs[0]

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File ~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:726, in MHA.forward(self, x, past_key_values, attention_mask, **kwargs)
    722         attn_output = self._forward_self_attn(x, attention_mask)
    723     else:
    724         # If `past_key_values` are supplied, it means that we might have cached values and
    725         # could take advantage of cross-attention
--> 726         attn_output = self._forward_cross_attn(x, past_key_values, attention_mask)
    727 # MQA / GQA
    728 else:
    729     # Regardless of `past_key_values` being supplied or not, it always use cross-attention
    730     # because `q` and `kv` lengths might be different
    731     attn_output = self._forward_cross_attn(x, past_key_values, attention_mask)

File ~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:704, in MHA._forward_cross_attn(self, x, past_key_values, key_padding_mask)
    695 if self.checkpointing:
    696     return torch.utils.checkpoint.checkpoint(
    697         self.inner_cross_attn,
    698         q,
   (...)
    701         causal=causal,
    702     )
--> 704 return self.inner_cross_attn(q, kv, key_padding_mask=key_padding_mask, causal=causal)

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/amp/autocast_mode.py:16, in autocast_decorator.<locals>.decorate_autocast(*args, **kwargs)
     13 @functools.wraps(func)
     14 def decorate_autocast(*args, **kwargs):
     15     with autocast_instance:
---> 16         return func(*args, **kwargs)

File /media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/amp/autocast_mode.py:16, in autocast_decorator.<locals>.decorate_autocast(*args, **kwargs)
     13 @functools.wraps(func)
     14 def decorate_autocast(*args, **kwargs):
     15     with autocast_instance:
---> 16         return func(*args, **kwargs)

File ~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:452, in CrossAttention.forward(self, q, kv, causal, key_padding_mask, **kwargs)
    448     causal_mask = cols > rows + seqlen_k - seqlen_q
    450     scores = scores.masked_fill(causal_mask, -10000.0)
--> 452 attention = torch.softmax(scores, dim=-1).to(v.dtype)
    453 attention = self.drop(attention)
    455 output = torch.einsum("bhts,bshd->bthd", attention, v)

OutOfMemoryError: CUDA out of memory. Tried to allocate 1.07 GiB. GPU 0 has a total capacty of 10.75 GiB of which 612.62 MiB is free. Including non-PyTorch memory, this process has 10.15 GiB memory in use. Of the allocated memory 9.78 GiB is allocated by PyTorch, and 179.66 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF

In [ ]:

gen(model, ds_train.with_format('pt')[0], tokenizer)

In [ ]:

In [ ]:

In [ ]: