detect_bs_text/nbs/01_detection_using_adapter_ft.ipynb

https://github.com/huggingface/peft/blob/main/examples/fp4_finetuning/finetune_fp4_opt_bnb_peft.py

In [1]:

from torch import optim
import lightning as pl
from matplotlib import pyplot as plt

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

In [2]:

import os

import torch
import torch.nn as nn
import transformers
from datasets import load_dataset
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, AutoConfig
import numpy as np
from tqdm.auto import tqdm
import pandas as pd
import warnings
from peft import LoraConfig, get_peft_model, IA3Config

In [3]:

plt.style.use('ggplot')
torch.set_float32_matmul_precision('medium')
warnings.filterwarnings("ignore", ".*does not have many workers.*")

In [4]:

# os.environ["CUDA_VISIBLE_DEVICES"] = "0"

model_name = "microsoft/phi-2"

# model = AutoModelForCausalLM.from_pretrained(
#     model_name,
#     # max_memory=max_memory,
#     quantization_config=BitsAndBytesConfig(
#         load_in_4bit=True,
#         llm_int8_threshold=6.0,
#         llm_int8_has_fp16_weight=False,
#         bnb_4bit_compute_dtype=torch.float16,
#         bnb_4bit_use_double_quant=True,
#         bnb_4bit_quant_type="nf4",
#     ),
#     torch_dtype=torch.float16,
#     trust_remote_code=True,
# )

In [5]:

# model_name = "TheBloke/phi-2-GPTQ"
model_name = "microsoft/phi-2"

def load_model():

    model = AutoModelForCausalLM.from_pretrained(
        model_name,
        # quantization_config=BitsAndBytesConfig(
        #     load_in_4bit=True,
        #     llm_int8_threshold=6.0,
        #     llm_int8_has_fp16_weight=False,
        #     bnb_4bit_compute_dtype=torch.float16,
        #     bnb_4bit_use_double_quant=True,
        #     bnb_4bit_quant_type="nf4",
        # ),
        torch_dtype=torch.float16,
        trust_remote_code=True,
    )


    # config = AutoConfig.from_pretrained(model_name, trust_remote_code=True,)
    # config.quantization_config['use_exllama'] = False
    # config.quantization_config['disable_exllama'] = True
    # model = AutoModelForCausalLM.from_pretrained(
    #     model_name,
    #     torch_dtype=torch.bfloat16,
    #     trust_remote_code=True,
    #     config=config,
    # )
    return model

In [6]:

base_model = load_model()
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True,)
tokenizer.pad_token = tokenizer.eos_token

Loading checkpoint shards: 100%|██████████| 2/2 [00:01<00:00,  1.86it/s]
Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.

In [7]:

def reset_model(base_model):
    # peft_config = LoraConfig(
    #     # task_type=TaskType.TOKEN_CLS, 
    #     target_modules=[ "fc2",  "Wqkv",],
    #     inference_mode=False, r=4, lora_alpha=4, 
    #     # lora_dropout=0.1, 
    #     # bias="all"
    # )
    peft_config = IA3Config(
        target_modules=[ "fc2",  "Wqkv",], 
            feedforward_modules=["fc2"],
            inference_mode=False,
    )
    model = get_peft_model(base_model, peft_config)
    model.config.use_cache = False
    return model

model = reset_model(base_model)

In [8]:

import json
MAX_LEN = 2000
samples = json.load(open("../samples.json"))

Helpers

In [9]:

# modified from https://github.dev/huggingface/evaluate/blob/8dfe05784099fb9af55b8e77793205a3b7c86465/measurements/perplexity/perplexity.py#L154

# from evaluate.measurements.perplexity import Perplexity
import evaluate
from evaluate import logging
from torch.nn import CrossEntropyLoss

# @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
def perplexity_compute(
    data, model, tokenizer, batch_size: int = 16, add_start_token: bool = True, device=None, max_length=None
):

    if device is not None:
        assert device in ["gpu", "cpu", "cuda"], "device should be either gpu or cpu."
        if device == "gpu":
            device = "cuda"
    else:
        device = "cuda" if torch.cuda.is_available() else "cpu"

    # model = AutoModelForCausalLM.from_pretrained(model_id)
    model = model.to(device)

    # tokenizer = AutoTokenizer.from_pretrained(model_id)

    # # if batch_size > 1 (which generally leads to padding being required), and
    # # if there is not an already assigned pad_token, assign an existing
    # # special token to also be the padding token
    # if tokenizer.pad_token is None and batch_size > 1:
    #     existing_special_tokens = list(tokenizer.special_tokens_map_extended.values())
    #     # check that the model already has at least one special token defined
    #     assert (
    #         len(existing_special_tokens) > 0
    #     ), "If batch_size > 1, model must have at least one special token to use for padding. Please use a different model or set batch_size=1."
    #     # assign one of the special tokens to also be the pad token
    #     tokenizer.add_special_tokens({"pad_token": existing_special_tokens[0]})

    # if add_start_token and max_length:
    #     # leave room for <BOS> token to be added:
    #     assert (
    #         tokenizer.bos_token is not None
    #     ), "Input model must already have a BOS token if using add_start_token=True. Please use a different model, or set add_start_token=False"
    #     max_tokenized_len = max_length - 1
    # else:
    max_tokenized_len = max_length

    encodings = tokenizer(
        data,
        add_special_tokens=False,
        padding=True,
        truncation=True if max_tokenized_len else False,
        max_length=max_tokenized_len,
        return_tensors="pt",
        return_attention_mask=True,
    ).to(device)

    encoded_texts = encodings["input_ids"]
    attn_masks = encodings["attention_mask"]

    # check that each input is long enough:
    if add_start_token:
        assert torch.all(torch.ge(attn_masks.sum(1), 1)), "Each input text must be at least one token long."
    else:
        assert torch.all(
            torch.ge(attn_masks.sum(1), 2)
        ), "When add_start_token=False, each input text must be at least two tokens long. Run with add_start_token=True if inputting strings of only one token, and remove all empty input strings."

    ppls = []
    loss_fct = CrossEntropyLoss(reduction="none")

    for start_index in logging.tqdm(range(0, len(encoded_texts), batch_size)):
        end_index = min(start_index + batch_size, len(encoded_texts))
        encoded_batch = encoded_texts[start_index:end_index]
        attn_mask = attn_masks[start_index:end_index]

        # if add_start_token:
        #     bos_tokens_tensor = torch.tensor([[tokenizer.bos_token_id]] * encoded_batch.size(dim=0)).to(device)
        #     encoded_batch = torch.cat([bos_tokens_tensor, encoded_batch], dim=1)
        #     attn_mask = torch.cat(
        #         [torch.ones(bos_tokens_tensor.size(), dtype=torch.int64).to(device), attn_mask], dim=1
        #     )

        labels = encoded_batch

        with torch.no_grad():
            out_logits = model(encoded_batch, attention_mask=attn_mask).logits
            # print(out_logits.shape)

        shift_logits = out_logits[..., :-1, :].contiguous()
        shift_labels = labels[..., 1:].contiguous()
        shift_attention_mask_batch = attn_mask[..., 1:].contiguous()

        perplexity_batch = torch.exp(
            (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch).sum(1)
            / shift_attention_mask_batch.sum(1)
        )
        # perplexity_batch = torch.exp(
        #     (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch)
        #     / shift_attention_mask_batch.sum(1)
        # )
        # print(perplexity_batch.shape)

        ppls += perplexity_batch.tolist()

    return {"perplexities": ppls, "mean_perplexity": torch.tensor(ppls).mean()}

In [ ]:

In [10]:

# perplexity_compute(
#     second_half, model, tokenizer
# )

Training

In [11]:

from torch.nn import functional as F
from torch.utils.data import DataLoader, TensorDataset

Lightning helpers

In [12]:

# def str2xya(s, tokenizer):
#     max_len = min(MAX_LEN, len(s))
#     input_ids = tokenizer(s, return_tensors="pt")["input_ids"][0]

#     pad = tokenizer.bos_token_id
#     data = []
#     for i in range(1, len(input_ids)):
#         x = input_ids[:i][-max_len:]
#         padding = max_len - len(x)
#         x = torch.tensor([pad]*padding + x.tolist())

#         labels = input_ids[i:i+1]
#         attention_mask = (x==pad)*1
#         data.append(dict(input_ids=x, labels=labels, attention_mask=attention_mask, return_loss=True))
        
#     return data

In [13]:

def eval(model, tokenizer, second_half):
    model.eval();
    with torch.no_grad():
        with model.disable_adapter():
            results = perplexity_compute(data=second_half, model=model, tokenizer=tokenizer, device='cuda')
        results2 = perplexity_compute(data=second_half, model=model, tokenizer=tokenizer, device='cuda')
    return dict(before=results['mean_perplexity'].item(), after=results2['mean_perplexity'].item())

Train

In [14]:

from datasets import Dataset


def compute_metrics(eval_prediction):
    return {}

Trainer docs

• https://huggingface.co/docs/transformers/v4.36.1/en/main_classes/trainer#transformers.Trainer

In [15]:

def learn_sample(sample):
    # device = 'cuda'
    # lr = 4e-3
    # epochs = 3
    # accum_steps = 1
    batch_size = 6
    verbose = True

    s = sample['text']
    first_half = s[:len(s)//2]
    second_half = s[len(s)//2:]
    ds_train = Dataset.from_dict(tokenizer([first_half]))
    ds_val = Dataset.from_dict(tokenizer([second_half]))

    os.environ['CUDA_VISIBLE_DEVICES']="1"
    model = reset_model(base_model)
    eval(model, tokenizer, second_half)

    # https://huggingface.co/docs/transformers/v4.36.1/en/main_classes/trainer#transformers.Trainer
    trainer = transformers.Trainer(
        model=model,
        train_dataset=ds_train,
        eval_dataset=ds_val,
        compute_metrics=compute_metrics, # without this it wont even give val loss
        args=transformers.TrainingArguments(
            # checkpoint='epoch',
            save_strategy='epoch',
            label_names=['labels',],
            per_device_train_batch_size=batch_size,
            gradient_accumulation_steps=3,
            warmup_steps=6,
            max_steps=20,
            learning_rate=2e-3,
            fp16=True,
            logging_steps=1,
            output_dir="outputs",
            log_level='error',
            # do_eval=True,
            evaluation_strategy="epoch",
            eval_steps=1,
            load_best_model_at_end=True,
            
            # disable_tqdm=not verbose,
        ),
        data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),
    )
    trainer._signature_columns = ['input_ids', 'attention_mask', 'labels',]
    model.config.use_cache = False  # silence the warnings. Please re-enable for inference!
    train_output = trainer.train()

    df_hist = pd.DataFrame(trainer.state.log_history)
    df_hist_epoch = df_hist.groupby('epoch').last().drop(columns=['step'])
    df_hist_step = df_hist.set_index('step').dropna(thresh=2, axis=1)
    if verbose:
        df_hist_epoch['loss'].plot()
        plt.twinx()
        df_hist_epoch['eval_loss'].plot(c='b', label='eval')
        plt.legend()
        plt.show()


    result_train = {f'train/{k}':v for k,v in eval(model, tokenizer, first_half).items()}
    result = eval(model, tokenizer, second_half)
    result['hist'] = df_hist_epoch
    result.update(result_train)
    return result

In [ ]:

data = []
for sample in samples:
    r = learn_sample(sample)
    print(sample['name'])
    print(dict(before=r['before'], after=r['after']))
    data.append(dict(**r, **sample))

In [25]:

# example training
df_hist = data[-1]['hist']#.groupby('epoch').last().dropna(axis=1).drop(columns=['step'])
df_hist['loss'].plot(label='train')
plt.twinx()
df_hist['eval_loss'].plot(c='b', label='eval')
plt.legend()
plt.show()

In [26]:

df_hist['learning_rate'].plot(logy=True)

Out [26]:

<Axes: xlabel='epoch'>

Perplexity

Perplexity measures how well a language model predicts a text sample. Lower is better

It’s calculated as the average number of bits per word a model needs to represent the same

https://huggingface.co/docs/transformers/perplexity https://thegradient.pub/understanding-evaluation-metrics-for-language-models/

The improvement column, is perplexity decrease

In [28]:

df_res = pd.DataFrame(data)
df_res['len'] = df_res.text.str.len()
df_res = df_res[['before', 'after', 'name', 'in_training', 'len']].set_index('name')
df_res['improvement%'] = (df_res['before'] - df_res['after'])/ df_res['before']
df_res['improvement'] = (df_res['before'] - df_res['after'])
df_res = df_res.sort_values('improvement%', ascending=False)
df_res

Out [28]:

	before	after	in_training	len	improvement%	improvement
name
wikipedia on LK-99	32.219017	28.852493	False	1038	0.104489	3.366524
Theory o. general relativity	26.952023	24.542513	True	1378	0.089400	2.409510
good_ml	28.347332	26.456573	False	1004	0.066700	1.890759
enron_email1	25.769749	24.390415	True	445	0.053525	1.379333
openai_board_ann	15.903965	15.173633	False	1191	0.045921	0.730332
Schmidhuber 2023 Subjective Novelty, Surprise	29.614954	28.470770	False	2654	0.038635	1.144184
email_to_fauci	25.089315	24.371374	False	1559	0.028615	0.717941
sokal hoax	15.966413	15.714754	True	2487	0.015762	0.251658
AI gen fake paper	7.632835	7.579506	False	2031	0.006987	0.053329
lorem ipsum	1.601658	1.595379	True	445	0.003921	0.006279
bad_ml	13.906106	13.862306	False	2345	0.003150	0.043800
I have a dream	2.127256	2.123436	True	848	0.001796	0.003820

In [29]:

print(df_res.to_markdown())

| name                                          |   before |    after | in_training   |   len |   improvement% |   improvement |
|:----------------------------------------------|---------:|---------:|:--------------|------:|---------------:|--------------:|
| wikipedia on LK-99                            | 32.219   | 28.8525  | False         |  1038 |     0.104489   |    3.36652    |
| Theory o. general relativity                  | 26.952   | 24.5425  | True          |  1378 |     0.0894     |    2.40951    |
| good_ml                                       | 28.3473  | 26.4566  | False         |  1004 |     0.0666997  |    1.89076    |
| enron_email1                                  | 25.7697  | 24.3904  | True          |   445 |     0.0535253  |    1.37933    |
| openai_board_ann                              | 15.904   | 15.1736  | False         |  1191 |     0.0459214  |    0.730332   |
| Schmidhuber 2023 Subjective Novelty, Surprise | 29.615   | 28.4708  | False         |  2654 |     0.0386353  |    1.14418    |
| email_to_fauci                                | 25.0893  | 24.3714  | False         |  1559 |     0.0286154  |    0.717941   |
| sokal hoax                                    | 15.9664  | 15.7148  | True          |  2487 |     0.0157617  |    0.251658   |
| AI gen fake paper                             |  7.63283 |  7.57951 | False         |  2031 |     0.00698672 |    0.0533285  |
| lorem ipsum                                   |  1.60166 |  1.59538 | True          |   445 |     0.00392053 |    0.00627935 |
| bad_ml                                        | 13.9061  | 13.8623  | False         |  2345 |     0.00314972 |    0.0438004  |
| I have a dream                                |  2.12726 |  2.12344 | True          |   848 |     0.00179583 |    0.00382018 |

DEBUG

In [21]:

from IPython.display import display, HTML, Markdown
import torch

@torch.no_grad()
def gen(model, inputs, tokenizer, clean=True):
    s = model.generate(
        input_ids=inputs["input_ids"][None, :].to(model.device),
        attention_mask=inputs["attention_mask"][None, :].to(model.device),
        use_cache=False,
        max_new_tokens=100,
        min_new_tokens=100,
        do_sample=False,
        early_stopping=False,
    )
    input_l = inputs["input_ids"].shape[0]
    tokenizer_kwargs=dict(clean_up_tokenization_spaces=clean, skip_special_tokens=clean)
    old = tokenizer.decode(
        s[0, :input_l], **tokenizer_kwargs
    )
    new = tokenizer.decode(
        s[0, input_l:], **tokenizer_kwargs
    )
    s_old = ""+old.replace('\n', '<br>')
    s_new =  '<b>' + new.replace('\n', '<br>')+ '<br><br><b/>'
    display(HTML(f"{s_old}{s_new}"))
    # print([old, new])

In [22]:

sample = samples[-1]
s = sample['text']
first_half = s[:len(s)//2]
second_half = s[len(s)//2:]
ds_train = Dataset.from_dict(tokenizer([first_half]))
ds_val = Dataset.from_dict(tokenizer([second_half]))

In [23]:

with model.disable_adapter():
    gen(model, ds_train.with_format('pt')[0], tokenizer)

/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/generation/utils.py:1421: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use and modify the model generation configuration (see https://huggingface.co/docs/transformers/generation_strategies#default-text-generation-configuration )
  warnings.warn(

The board of directors of OpenAI, Inc., the 501(c)(3) that acts as the overall governing body for all OpenAI activities, today announced that Sam Altman will depart as CEO and leave the board of directors. Mira Murati, the company’s chief technology officer, will serve as interim CEO, effective immediately.

A member of OpenAI’s leadership team for five years, Mira has played a critical role in OpenAI’s evolution into a global AI leader. She brings a unique skill set, understanding of the company’s values, operations, and business, and already leads the company’s research, product, and sa

Mira Murati, the chief technology officer of OpenAI, was known for her exceptional problem-solving skills. She was always able to find innovative solutions to complex challenges. One day, she was faced with a dilemma. The company had developed a new AI model that had the potential to revolutionize the field of natural language processing. However, the model required a significant amount of computational power to train effectively.

Mira knew that the company's current infrastructure was not capable of

In [24]:

gen(model, ds_train.with_format('pt')[0], tokenizer)

The board of directors of OpenAI, Inc., the 501(c)(3) that acts as the overall governing body for all OpenAI activities, today announced that Sam Altman will depart as CEO and leave the board of directors. Mira Murati, the company’s chief technology officer, will serve as interim CEO, effective immediately.

A member of OpenAI’s leadership team for five years, Mira has played a critical role in OpenAI’s evolution into a global AI leader. She brings a unique skill set, understanding of the company’s values, operations, and business, and already leads the company’s research, product, and sa

Mira: Thank you all for the warm welcome. I'm honored to be stepping into the role of interim CEO. As we navigate this transition, I want to assure you that OpenAI's mission and values will remain at the forefront of our decision-making.

John: Mira, we're confident in your abilities and believe you'll lead OpenAI to even greater heights. Can you tell us more about your plans for the company?

Mira: Absolutely, John

In [ ]: