wassname/detect_bs_text
1
0 Fork 0
mirror of https://github.com/wassname/detect_bs_text.git synced 2026-10-01 12:01:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

tidy

Browse Source
This commit is contained in:
wassname committed 2024-01-03 13:25:38 +08:00
1 parent 7f27b90159
commit 421c3a8ee0
9 files changed
+1002 -13679
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

README.md
+29 -1
View File
@@ -16,7 +16,33 @@ If yes/yes then it's may be suprising new information. If either is not true, th
See [main.ipynb](main.ipynb) for the code and results.
# Results
# Results using adapter fine tuning
I fine tuned the model on the first half of a text, then tested on the second half. I measure how much it learned by the perplexity decrease. The rows with a high perplexity are unpredictable, and the ones with a higher improvement are learnable. Unpredictable and learnable text is not BS.
| name | before | after | in_training | len | improvement% | improvement |
|:----------------------------------------------|---------:|---------:|:--------------|------:|---------------:|--------------:|
| wikipedia on LK-99 | 32.219 | 28.8525 | False | 1038 | 0.104489 | 3.36652 |
| Theory o. general relativity | 26.952 | 24.5425 | True | 1378 | 0.0894 | 2.40951 |
| good_ml | 28.3473 | 26.4566 | False | 1004 | 0.0666997 | 1.89076 |
| enron_email1 | 25.7697 | 24.3904 | True | 445 | 0.0535253 | 1.37933 |
| openai_board_ann | 15.904 | 15.1736 | False | 1191 | 0.0459214 | 0.730332 |
| Schmidhuber 2023 Subjective Novelty, Surprise | 29.615 | 28.4708 | False | 2654 | 0.0386353 | 1.14418 |
| email_to_fauci | 25.0893 | 24.3714 | False | 1559 | 0.0286154 | 0.717941 |
| sokal hoax | 15.9664 | 15.7148 | True | 2487 | 0.0157617 | 0.251658 |
| AI gen fake paper | 7.63283 | 7.57951 | False | 2031 | 0.00698672 | 0.0533285 |
| lorem ipsum | 1.60166 | 1.59538 | True | 445 | 0.00392053 | 0.00627935 |
| bad_ml | 13.9061 | 13.8623 | False | 2345 | 0.00314972 | 0.0438004 |
| I have a dream | 2.12726 | 2.12344 | True | 848 | 0.00179583 | 0.00382018 |
For example the wikipedia extract `wikipedia on LK-99 ` is unpredictable (high before perplexity) and is learnable (high improvement in perplexity). That makes sense as it's a new topic. In contrast `lorem ipsum` has a low perplexity, meaning it's predictalbe or memorizable. That makes sense as this text was likely in the training corpus. The `AI gen fake paper ` has a low perplexity because it's predictable, even thought it is new.
See more in [01_detection_using_adapter_ft.ipynb](01_detection_using_adapter_ft.ipynb)
# Results using prompting
When using microsoft/phi-2 we get this amount of perplexity reduction by including a summary of the key learnings
@@ -35,6 +61,8 @@ When using microsoft/phi-2 we get this amount of perplexity reduction by includi
As you can see, some of these are probobly in the training set
See more in [02_detection_using_tldr_prompt.ipynb](02_detection_using_tldr_prompt.ipynb)
# Citing
If you like our work and end up using this code for your reseach give us a shout-out by citing or acknowledging
nbs/01_detection_using_adapter_ft.ipynb
+972
View File
@@ -0,0 +1,972 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"https://github.com/huggingface/peft/blob/main/examples/fp4_finetuning/finetune_fp4_opt_bnb_peft.py"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"from torch import optim\n",
"import lightning as pl\n",
"from matplotlib import pyplot as plt"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"import torch\n",
"import torch.nn as nn\n",
"import transformers\n",
"from datasets import load_dataset\n",
"from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, AutoConfig\n",
"import numpy as np\n",
"from tqdm.auto import tqdm\n",
"import pandas as pd\n",
"import warnings\n",
"from peft import LoraConfig, get_peft_model, IA3Config"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"plt.style.use('ggplot')\n",
"torch.set_float32_matmul_precision('medium')\n",
"warnings.filterwarnings(\"ignore\", \".*does not have many workers.*\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
"\n",
"model_name = \"microsoft/phi-2\"\n",
"\n",
"# model = AutoModelForCausalLM.from_pretrained(\n",
"# model_name,\n",
"# # max_memory=max_memory,\n",
"# quantization_config=BitsAndBytesConfig(\n",
"# load_in_4bit=True,\n",
"# llm_int8_threshold=6.0,\n",
"# llm_int8_has_fp16_weight=False,\n",
"# bnb_4bit_compute_dtype=torch.float16,\n",
"# bnb_4bit_use_double_quant=True,\n",
"# bnb_4bit_quant_type=\"nf4\",\n",
"# ),\n",
"# torch_dtype=torch.float16,\n",
"# trust_remote_code=True,\n",
"# )\n",
"\n",
"\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"# model_name = \"TheBloke/phi-2-GPTQ\"\n",
"model_name = \"microsoft/phi-2\"\n",
"\n",
"def load_model():\n",
"\n",
" model = AutoModelForCausalLM.from_pretrained(\n",
" model_name,\n",
" # quantization_config=BitsAndBytesConfig(\n",
" # load_in_4bit=True,\n",
" # llm_int8_threshold=6.0,\n",
" # llm_int8_has_fp16_weight=False,\n",
" # bnb_4bit_compute_dtype=torch.float16,\n",
" # bnb_4bit_use_double_quant=True,\n",
" # bnb_4bit_quant_type=\"nf4\",\n",
" # ),\n",
" torch_dtype=torch.float16,\n",
" trust_remote_code=True,\n",
" )\n",
"\n",
"\n",
" # config = AutoConfig.from_pretrained(model_name, trust_remote_code=True,)\n",
" # config.quantization_config['use_exllama'] = False\n",
" # config.quantization_config['disable_exllama'] = True\n",
" # model = AutoModelForCausalLM.from_pretrained(\n",
" # model_name,\n",
" # torch_dtype=torch.bfloat16,\n",
" # trust_remote_code=True,\n",
" # config=config,\n",
" # )\n",
" return model\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Loading checkpoint shards: 100%|██████████| 2/2 [00:01<00:00, 1.86it/s]\n",
"Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
]
}
],
"source": [
"base_model = load_model()\n",
"tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True,)\n",
"tokenizer.pad_token = tokenizer.eos_token"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"def reset_model(base_model):\n",
" # peft_config = LoraConfig(\n",
" # # task_type=TaskType.TOKEN_CLS, \n",
" # target_modules=[ \"fc2\", \"Wqkv\",],\n",
" # inference_mode=False, r=4, lora_alpha=4, \n",
" # # lora_dropout=0.1, \n",
" # # bias=\"all\"\n",
" # )\n",
" peft_config = IA3Config(\n",
" target_modules=[ \"fc2\", \"Wqkv\",], \n",
" feedforward_modules=[\"fc2\"],\n",
" inference_mode=False,\n",
" )\n",
" model = get_peft_model(base_model, peft_config)\n",
" model.config.use_cache = False\n",
" return model\n",
"\n",
"model = reset_model(base_model)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"MAX_LEN = 2000\n",
"samples = json.load(open(\"../samples.json\"))\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Helpers"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"# modified from https://github.dev/huggingface/evaluate/blob/8dfe05784099fb9af55b8e77793205a3b7c86465/measurements/perplexity/perplexity.py#L154\n",
"\n",
"# from evaluate.measurements.perplexity import Perplexity\n",
"import evaluate\n",
"from evaluate import logging\n",
"from torch.nn import CrossEntropyLoss\n",
"\n",
"# @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)\n",
"def perplexity_compute(\n",
" data, model, tokenizer, batch_size: int = 16, add_start_token: bool = True, device=None, max_length=None\n",
"):\n",
"\n",
" if device is not None:\n",
" assert device in [\"gpu\", \"cpu\", \"cuda\"], \"device should be either gpu or cpu.\"\n",
" if device == \"gpu\":\n",
" device = \"cuda\"\n",
" else:\n",
" device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
"\n",
" # model = AutoModelForCausalLM.from_pretrained(model_id)\n",
" model = model.to(device)\n",
"\n",
" # tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
"\n",
" # # if batch_size > 1 (which generally leads to padding being required), and\n",
" # # if there is not an already assigned pad_token, assign an existing\n",
" # # special token to also be the padding token\n",
" # if tokenizer.pad_token is None and batch_size > 1:\n",
" # existing_special_tokens = list(tokenizer.special_tokens_map_extended.values())\n",
" # # check that the model already has at least one special token defined\n",
" # assert (\n",
" # len(existing_special_tokens) > 0\n",
" # ), \"If batch_size > 1, model must have at least one special token to use for padding. Please use a different model or set batch_size=1.\"\n",
" # # assign one of the special tokens to also be the pad token\n",
" # tokenizer.add_special_tokens({\"pad_token\": existing_special_tokens[0]})\n",
"\n",
" # if add_start_token and max_length:\n",
" # # leave room for <BOS> token to be added:\n",
" # assert (\n",
" # tokenizer.bos_token is not None\n",
" # ), \"Input model must already have a BOS token if using add_start_token=True. Please use a different model, or set add_start_token=False\"\n",
" # max_tokenized_len = max_length - 1\n",
" # else:\n",
" max_tokenized_len = max_length\n",
"\n",
" encodings = tokenizer(\n",
" data,\n",
" add_special_tokens=False,\n",
" padding=True,\n",
" truncation=True if max_tokenized_len else False,\n",
" max_length=max_tokenized_len,\n",
" return_tensors=\"pt\",\n",
" return_attention_mask=True,\n",
" ).to(device)\n",
"\n",
" encoded_texts = encodings[\"input_ids\"]\n",
" attn_masks = encodings[\"attention_mask\"]\n",
"\n",
" # check that each input is long enough:\n",
" if add_start_token:\n",
" assert torch.all(torch.ge(attn_masks.sum(1), 1)), \"Each input text must be at least one token long.\"\n",
" else:\n",
" assert torch.all(\n",
" torch.ge(attn_masks.sum(1), 2)\n",
" ), \"When add_start_token=False, each input text must be at least two tokens long. Run with add_start_token=True if inputting strings of only one token, and remove all empty input strings.\"\n",
"\n",
" ppls = []\n",
" loss_fct = CrossEntropyLoss(reduction=\"none\")\n",
"\n",
" for start_index in logging.tqdm(range(0, len(encoded_texts), batch_size)):\n",
" end_index = min(start_index + batch_size, len(encoded_texts))\n",
" encoded_batch = encoded_texts[start_index:end_index]\n",
" attn_mask = attn_masks[start_index:end_index]\n",
"\n",
" # if add_start_token:\n",
" # bos_tokens_tensor = torch.tensor([[tokenizer.bos_token_id]] * encoded_batch.size(dim=0)).to(device)\n",
" # encoded_batch = torch.cat([bos_tokens_tensor, encoded_batch], dim=1)\n",
" # attn_mask = torch.cat(\n",
" # [torch.ones(bos_tokens_tensor.size(), dtype=torch.int64).to(device), attn_mask], dim=1\n",
" # )\n",
"\n",
" labels = encoded_batch\n",
"\n",
" with torch.no_grad():\n",
" out_logits = model(encoded_batch, attention_mask=attn_mask).logits\n",
" # print(out_logits.shape)\n",
"\n",
" shift_logits = out_logits[..., :-1, :].contiguous()\n",
" shift_labels = labels[..., 1:].contiguous()\n",
" shift_attention_mask_batch = attn_mask[..., 1:].contiguous()\n",
"\n",
" perplexity_batch = torch.exp(\n",
" (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch).sum(1)\n",
" / shift_attention_mask_batch.sum(1)\n",
" )\n",
" # perplexity_batch = torch.exp(\n",
" # (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch)\n",
" # / shift_attention_mask_batch.sum(1)\n",
" # )\n",
" # print(perplexity_batch.shape)\n",
"\n",
" ppls += perplexity_batch.tolist()\n",
"\n",
" return {\"perplexities\": ppls, \"mean_perplexity\": torch.tensor(ppls).mean()}"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"# perplexity_compute(\n",
"# second_half, model, tokenizer\n",
"# )"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Training"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"from torch.nn import functional as F\n",
"from torch.utils.data import DataLoader, TensorDataset"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Lightning helpers"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"\n",
"\n",
"\n",
"\n",
"# def str2xya(s, tokenizer):\n",
"# max_len = min(MAX_LEN, len(s))\n",
"# input_ids = tokenizer(s, return_tensors=\"pt\")[\"input_ids\"][0]\n",
"\n",
"# pad = tokenizer.bos_token_id\n",
"# data = []\n",
"# for i in range(1, len(input_ids)):\n",
"# x = input_ids[:i][-max_len:]\n",
"# padding = max_len - len(x)\n",
"# x = torch.tensor([pad]*padding + x.tolist())\n",
"\n",
"# labels = input_ids[i:i+1]\n",
"# attention_mask = (x==pad)*1\n",
"# data.append(dict(input_ids=x, labels=labels, attention_mask=attention_mask, return_loss=True))\n",
" \n",
"# return data\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"def eval(model, tokenizer, second_half):\n",
" model.eval();\n",
" with torch.no_grad():\n",
" with model.disable_adapter():\n",
" results = perplexity_compute(data=second_half, model=model, tokenizer=tokenizer, device='cuda')\n",
" results2 = perplexity_compute(data=second_half, model=model, tokenizer=tokenizer, device='cuda')\n",
" return dict(before=results['mean_perplexity'].item(), after=results2['mean_perplexity'].item())\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Train"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"from datasets import Dataset\n",
"\n",
"\n",
"def compute_metrics(eval_prediction):\n",
" return {}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Trainer docs\n",
"\n",
"- https://huggingface.co/docs/transformers/v4.36.1/en/main_classes/trainer#transformers.Trainer"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"def learn_sample(sample):\n",
" # device = 'cuda'\n",
" # lr = 4e-3\n",
" # epochs = 3\n",
" # accum_steps = 1\n",
" batch_size = 6\n",
" verbose = True\n",
"\n",
" s = sample['text']\n",
" first_half = s[:len(s)//2]\n",
" second_half = s[len(s)//2:]\n",
" ds_train = Dataset.from_dict(tokenizer([first_half]))\n",
" ds_val = Dataset.from_dict(tokenizer([second_half]))\n",
"\n",
" os.environ['CUDA_VISIBLE_DEVICES']=\"1\"\n",
" model = reset_model(base_model)\n",
" eval(model, tokenizer, second_half)\n",
"\n",
" # https://huggingface.co/docs/transformers/v4.36.1/en/main_classes/trainer#transformers.Trainer\n",
" trainer = transformers.Trainer(\n",
" model=model,\n",
" train_dataset=ds_train,\n",
" eval_dataset=ds_val,\n",
" compute_metrics=compute_metrics, # without this it wont even give val loss\n",
" args=transformers.TrainingArguments(\n",
" # checkpoint='epoch',\n",
" save_strategy='epoch',\n",
" label_names=['labels',],\n",
" per_device_train_batch_size=batch_size,\n",
" gradient_accumulation_steps=3,\n",
" warmup_steps=6,\n",
" max_steps=20,\n",
" learning_rate=2e-3,\n",
" fp16=True,\n",
" logging_steps=1,\n",
" output_dir=\"outputs\",\n",
" log_level='error',\n",
" # do_eval=True,\n",
" evaluation_strategy=\"epoch\",\n",
" eval_steps=1,\n",
" load_best_model_at_end=True,\n",
" \n",
" # disable_tqdm=not verbose,\n",
" ),\n",
" data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),\n",
" )\n",
" trainer._signature_columns = ['input_ids', 'attention_mask', 'labels',]\n",
" model.config.use_cache = False # silence the warnings. Please re-enable for inference!\n",
" train_output = trainer.train()\n",
"\n",
" df_hist = pd.DataFrame(trainer.state.log_history)\n",
" df_hist_epoch = df_hist.groupby('epoch').last().drop(columns=['step'])\n",
" df_hist_step = df_hist.set_index('step').dropna(thresh=2, axis=1)\n",
" if verbose:\n",
" df_hist_epoch['loss'].plot()\n",
" plt.twinx()\n",
" df_hist_epoch['eval_loss'].plot(c='b', label='eval')\n",
" plt.legend()\n",
" plt.show()\n",
"\n",
"\n",
" result_train = {f'train/{k}':v for k,v in eval(model, tokenizer, first_half).items()}\n",
" result = eval(model, tokenizer, second_half)\n",
" result['hist'] = df_hist_epoch\n",
" result.update(result_train)\n",
" return result\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"data = []\n",
"for sample in samples:\n",
" r = learn_sample(sample)\n",
" print(sample['name'])\n",
" print(dict(before=r['before'], after=r['after']))\n",
" data.append(dict(**r, **sample))"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAlQAAAG0CAYAAAD0NLk2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAACBwElEQVR4nO3dd3gU1dvG8e+ZbMqmN0IS6V2lqnRRiq8gogLSBGwUlaCiP0GlCAEJCihFQAWJhS4iKE1EKYoggoIFERGDUkKAQDa9bWbeP9ZEYhJIL7vP57pywU6mnCeTTe6cOXNGGYZhIIQQQgghik2r6AYIIYQQQlR1EqiEEEIIIUpIApUQQgghRAlJoBJCCCGEKCEJVEIIIYQQJSSBSgghhBCihCRQCSGEEEKUkAQqIYQQQogSkkAlhBBCCFFCpopuQGlKSkoiMzOzoptR5vz8/IiLi6voZpQLqdV+OVK9Uqv9cqR6y6JWZ2dnPD09S3WfFcWuAlVmZiapqakV3YwypZQCIC0tDXt/apDUar8cqV6p1X45Ur2OVGtxySU/IYQQQogSkkAlhBBCCFFCEqiEEEIIIUpIApUQQgghRAnZ1aB0IYQQoixZrVZSUlJyXqemppKRkVGBLSo/JanV3d0dk8m+I4d9VyeEEEKUEqvVSnJyMl5eXmia7QKPs7OzQ0zXA8WvVdd1EhMT8fDwsOtQJZf8hBBCiEJISUnJFaZE4WiahpeXV66ePXsk3xVCCCFEIUmYKh5H+LrZf4VCCCGEEGVMApUQQgghRAlJoBJCCCGEKKFiDbfftm0bmzZtwmKxULt2bYYNG0aDBg3yXTc8PJyjR4/mWd6qVSvGjx8PwKJFi/jqq69yfb5FixZMnDixOM0TQgghRDkxm834+PgQExNT0U2pUEUOVPv27WPZsmWMHDmShg0bsmXLFiIiIpg3bx4+Pj551h87dixWqzXndWJiIuPGjaN9+/a51mvZsiVhYWH/NqwS3FppGHDqlBMmE5hMBs7O//7r5GT795/nRRb/GOnp4OKS8+BJIYQQQlQ9RU4tmzdvplu3bnTp0gWAkSNHcujQIXbt2kXv3r3zrO/p6Znr9d69e3F1daVdu3a5G2Iy4evrW6g2ZGZm5poLQ9M03NzcAEo1mFit0KFD9auukx2sTCbjqsHryuUmE5iUFdPls7hcPsvADkf4v7n3oZydr9mm7PocIYBJrfbLkeqVWoX4lz1/bxQpUFmtVqKionIFJ03TaNasGcePHy/UPnbu3EmHDh1yAlC2o0ePMmLECDw8PGjatCmDBg3Cy8sr331s2LCBdevW5bzu2LEjY8aMwc/PryjlXFNqKri724JVZqatx+q/srIUWVkARf0mcQUaAY34ZmNrDtz4Bo3DX0QV8tbS4ODgIh6v6pJa7Zcj1Su1Vn2pqak4X/GHr2FAcjLAtf8YLm3u7kW7QmI2mzGbzWiaRlZWFsnJyWRkZODv709KSgppaWk562Z3cFy+fBld1zGbzbi5ueHk5ISu62RkZJCUlJSzvpOTE0Cur01+XFxcCAkJKVqhVUiRAlVCQgK6rufpSfL19SU6Ovqa2584cYLTp08zatSoXMtbtmxJ27ZtCQoKIiYmhtWrVzNjxgwiIiLynbuiT58+9OrVK+d19jpxcXG5vilKw4kT//4/K8sWrLKyFJmZYLX++6/VeuVryMz87zJF5umzZHz9JdbzsWTqJqzegbz31wP8diaQiA8a8UrgKzj1H3bV9iilCA4OJiYmBiO/hGdHpFb75Uj1Sq32IyMjI9fVkZQURcOGFRMQ/vjjHO7uhfsae3p64urqisViwWq14uLigq+vL5cuXSI1NRUXFxcSExNz1jebzWRkZJCeng7YgpDFYkEphWEY+Pj44O7uTnx8PPDvEJ1rzaKekZHBuXPnci1zc3Mr9c6QilKuA5V27txJrVq18gxg79ixY87/a9WqRe3atXnqqaf49ddfadasWZ79ODs7F5iEy/JNrGng6gpQtGMYlssYH3+AsX8XOAH1zKh7HkB1vZv638P998OqU3146KPBXO+3Ea3bPdfep2HY5Q+s/Eit9suR6pVaRUXx9PTk0qVLOYEnO0S5u7uTlJREtWrVcHJyIst2uQWz2ZwrYCXbuuFyHj2TmJiIj49PTqAqCnv+vihSoPL29kbTNCwWS67lFovlmuOf0tLS2Lt3LwMHDrzmcapXr46XlxcxMTH5BqqqwsjMxNixEWPzWkhPBUB1vAPV90GUty2Rt2uXQc+eqWzdaublY8+yYs1TKN8A1M0dKrLpQgghrsFsNjh58mKuG6/K89iFYTKZ0DSNgICAXMuVUmRmZmK1WrFarZjNZpKSknBxcUHTNFJTU3PWdXFxwcvLC5PJhFIq14c9B6SiKlKgMplM1KtXjyNHjtCmTRvA9tDDI0eO0KNHj6tuu3//fqxWK506dbrmcS5dukRSUlKV7gY0fvkefc1SuPDPpdC6jdAeeAxVt1GedSdOTODLL93YE9uOnRc60G3p62hePqhGN5Zzq4UQQhSWUuDhAZmZlTdUZA8Cv3z5ck4PVLbsMJSampoTqMxmM+np6Tmfc3JyIiAggOTkZFJSUsjIyMDFxaVK/34uK0We2LNXr17s2LGD3bt3c+bMGZYuXUp6ejqdO3cGYOHChaxatSrPdjt37qR169Z5BpqnpaWxfPlyjh8/zoULF/jll1+YNWsWwcHBtGjRonhVVSDjfDRZb0xDf2OaLUx5+6IefQbtxVn5himAOnWyGD7c1qU6PWo8mRk6+qLpGNGnyrPpQggh7IzVasUwjJxLeld+6LoO2AKVyWTC2dkZs9mc6yHG2cNrEhISsFqtZGVl5QxCF7kVeQxVhw4dSEhIYO3atVgsFurUqcOECRNyLvnFxsbmuS0yOjqaY8eOMWnSpDz70zSNU6dO8dVXX5GcnIy/vz/Nmzdn4MCB17xjoDIx0lIwNq/F+HIjZFnByYS64x7U3QNRZvdrbv/004msXWvmz0vBrEgJ41FtAfr8qWjjZ6F8A665vRBCCPFfhmGQlJSEt7c3YBsYrmkaLi4u6LpOamoqWVlZZGRk5Pwev/LmLqvVilIKDw+PnEuDHh4eFVFKpacMO7oAGhcXl+u6b3kwdB3ju68wPn4f4uNsC5vehDZwBCq4RpH2tWyZO+PH++Ljk8Weux/C1/I71KiD9vyrOaFMKUVISAjnzp2z+2vXUqv9cqR6pVb7kZCQkBNMsmUP1K7sPDw8cHd3x2Qyoes6mZmZJCUlkZGRAYC7uzu+vr6kpKTkGSft4eGBp6cnmqaRnp5Oamoqfn5+Oee5sDOl5/f1M5vNdnP5sOKnI6/CjL/+QF/zDvx5zLYgKARtwAhofkuxJi8bPDiFDz7w4NgxZ+Yzlyneg+HMX+hvvYL29GSUqer02AkhhKg8kpOTc+7Wy09KSkquS335bXtleLyy8yI1NbXcOzMqI3k4cjEYCRb0DxagzxhrC1Oubqi+D6OFL0S1aF3smWBNJpgyJQGAD9ZW4+R9r4KrGX77CeP9NzD+ud4thBBCiMpFAlURGFYr+pefok8ahfHNF2AYqHad0aa/hXbX/YV6dMy13HZbOt26pWG1Kqa/2xTtiRfAycl2WXHD8lKoQgghhBClTQJVIRlHD6NPG4PxYSSkJkOt+mgvzEQb/r9SHzQ+eXICJpPBF1+4sedyO9SDT9rasO1j9J1bSvVYQgghhCg5GUN1DcbFGPS178KP+20LPL1RfR9CdeyG0srm1tEGDaw8/HAykZGeTJvmw+efd0PFxWJ8uhJ99WJS6taHuk3K5NhCCCGEKDrpoboK41QU+uTRtjClaahu96BFvI3W6c4yC1PZnn02EV9fnd9+c2b1anfU3QNQt/UAw+Dy7EkYfxwt0+MLIYTIS5exrMXiCF83CVRXU6MO1KwL17dAm/wG2qCRKHfPcjm0n5/Bs8/anqU0e7YXSUkaavDjqBZtMDLSyVrwMsa5M+XSFiGEELapBRITEx0iHJQmXddJTEzE3f3aczJWZXLJ7yqUpqE9Ew5mj2LfuVcSDz+czAcfeBAVZWLBAk8mTEhEPTYOpzemkvH7EfT54bYZ2H39y71tQgjhaEwmEx4eHiQlJeUsc3FxyZnLyd6VpFYPDw9MJvuOHPZdXSkorx6p/Dg7w+TJ8TzySADvvOPJkCEp1KnjRuCUuUQ/8xBcOIf+xlS0ca8UajZ2IYQQJWMymXImp7T3iUyv5Ei1Fpdc8qvk7rgjnU6d0snIUERE2N7ETj5+OD0zFbx84PRJ9LdfxbBW/pl6hRBCCHslgaqSUwqmTIlH0wy2bDHz3Xe2ua5UUAja05PB1Q2O/oixbKH81SCEEEJUEAlUVcD111t54AHbIwGmTPEmezykqtMQ7fEXQNMwvt2F8cmKCmylEEII4bgkUFURzz+fiKenzs8/u7D8ignTVbObUQ+OBsDY+hH67q0V1EIhhBDCcUmgqiICA3XGjLHdWTJhAqSk/HvXoXbr/6HueQAAY9USjOxJSIUQQghRLiRQVSHDhydRq5aV6GhYtMgj1+fUPYNQne4EQ0df8hrGn8cqqJVCCCGE45FAVYW4usKkSbbJPt96y5OzZ/89fUop1JBR0OwWyMxAX/gyRszZimqqEEII4VAkUFUxd9+dRqdOkJamePVV71yfU05OaI8/D3UaQlIi+vxwjPi4CmqpEEII4TgkUFUxSsHcuaCUwfr17hw+7Jz7865uaE+9BNWCIfY8+hvTMJITK6i1QgghhGOQQFUF3Xwz9O+fCkB4uA//nX5KefvaHpnj5QOn/kR/YQT6R+9hWC6Vf2OFEEIIByCBqop68cVEzGad7793YeNGtzyfV0GhaGPCbQ94Tk/F2L4BffxI9GULMS5El3t7hRBCCHsmgaqKCg7WGT3aNo3CjBnepKbmXUfVro82eb7tEmCD68FqxdizHX1SGPqS2Rinosq51UIIIYR9kkBVhT3Line truncated
"text/plain": [
"<Figure size 640x480 with 2 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# example training\n",
"df_hist = data[-1]['hist']#.groupby('epoch').last().dropna(axis=1).drop(columns=['step'])\n",
"df_hist['loss'].plot(label='train')\n",
"plt.twinx()\n",
"df_hist['eval_loss'].plot(c='b', label='eval')\n",
"plt.legend()\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<Axes: xlabel='epoch'>"
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAG0CAYAAAD6ncdZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABEh0lEQVR4nO3dd3hUVeLG8e+5SQiEBEIPvUjvoAKiKEUgKmtfVMB1VbChrtgXFUXFjmUta8NV1BX5ISqIBqXZUGGlC4JSBQyEEgKEltzz+2NMFGkJJDlzZ97P8+yzSZhM3stlnDf3nmKstRYRERGRgPBcBxAREREpDJUXERERCRSVFxEREQkUlRcREREJFJUXERERCRSVFxEREQkUlRcREREJFJUXERERCRSVFxEREQmUWNcBisvWrVvJyclxHaNYValShYyMDNcxSkw0Ha+ONTJF07FCdB2vjvXYxcbGUqFChYI9tsh/epjIyclh3759rmMUG2MMEDrOaNjhIZqOV8camaLpWCG6jlfHWvJ020hEREQCReVFREREAkXlRURERAJF5UVEREQCReVFREREAkXlRURERAJF5UVEREQCReVFREREAkXlRURERAJF5UVEREQCReVFREREAkXlRURERAJF5UWinvV97O5s1zFERKSAInZXaZGCsNk78Z97AH7+EVodj3dqb2h5PCYmxnU0ERE5hIgpL2lpaUyePJlatWpxyy23uI4jAWB37sB/5j5YuSz0hQWz8RfMhuRKmFNOx5zSE1OpqtOMIiJyoIgpL6mpqaSmprqOIQFhd2ThP3UvrFkOZZPw/n4D9qcl2JlTIXMz9qN3sZPGQov2oasxrU7AxEbMy0VEJND0X2OJOnb7Nvwn74G1qyCpPN7N92Nq1ce07YQ9dwB23rfYLybDjwtg0ff4i76H8hUxJ/cIXY2pkuL6EEREoprKi0QVm7UVf+Q9sH4NlEvGu+VBTI06+X9u4uIwJ3aBE7tgN67HfvkZ9uspsG0L9uP/w34yDpq1DV2NadNBV2NERBzQf3klatjMzaHikr4WkiuGiktKrUM+3lStgbngMuw5/WD+LPwvJsPiebB4Lv7iuZBUHnPy6ZguPTFVa5TcgYiIRDmVF4kKdssm/JF3w8b1ULFyqLgUsHCY2Dg4/mRijj8Zm5GO/SrvasxWbNp72LT3oFkbTJfemLYdMXFxxXw0IiLRTeVFIp7dvDFUXDLSoVLVUHE5ynErpkoK5rxLsX+5JDQ76cvJ8MNcWDIfu2Q+NrEcpnMPTJdemJSaRXwkIiICKi8S4WxGeqi4bN4IVVLwbhmBqVTlmJ/XxMZC+5OIaX8SdtMG7NdTsF99BplbsJ++j/30fWjSKlRi2p+EiStVBEcjIiKg8iIRzG5YHyouWzdBtZqhKy4VKhX5zzGVq2HO6Y/tczEs/F9obMyiObB0IXbpQmzZJEzn7phuZ2mmkohIEVB5kYhkf10bKi7btkD12ng3P4BJrlisP9PExEDbjsS07YjdkoH96rerMVs3YT/7EDtlArQ+Ea/HX6Bpa4wxxZpHRCRSqbxIxLHr1uA/eTdkZULNuqF1XMpVKNEMpmIVzNmXYPv0hYVz8Kd/FBobM38W/vxZUL02psdfMJ26YuJLl2g2EZGgU3mRiGLXrgxNh96RBbXr4w15AJNUzlke48VAmxOJaXMi9te12OkfYWdOg19/wb71Anb8G5hTemG6nYmpXM1ZThGRIFF5kYhhVy/Hf2oY7NwOdRviDRmOKZvkOlY+U70Wpt812HMvxc6cgp02CTLSQwN8P/sQ2nTA69EHmrZ2HVVEJKypvEhEsCt/wn96GGTvhPqN8W66D5OQ6DrWQZmEspjTz8F27xO6pTRtYmjxu3nf4s/7FmrWZcf5A7BN20KpeNdxRUTCjsqLBJ5d/mNod+hd2XBcU7x/3Icpk+A61hHtf0vpF+y0SdhvpsG61Wx9dgQkJIZW7+12lna3FhH5A5UXCTT702L8Z4bDnl3QuAXeDcMwpcu4jlVopnptTP9rsOcNgK+nYj5PI3fDOuzk97GffghtO4RmKTVuqVlKIhL1VF4ksOzShfj/uh/27oGmrfGuvzvwM3dMQiKm17mkDLiK9Z9+hD91AiyZD3O/xZ8buqVkevwF0+E0TLxuKYlIdFJ5kUCyi+fiPz8C9u6F5u3wBg/FRND4EBMTg9e2A6bNidh1a0KzlL6ZDutWY0c/h33vjdDqvV3PLJIVg0VEgkTlRQLHLvwe/4WHIGcftDoB79o7I3r5fVOzDmbAddjz/ob9+rPQLKXNG0ObQk5+H9p1wut5NqZhc9dRRURKhMqLBIo/bxb+iw9DTg607Yh39e2hXZ+jgCmbiOl1Hvb0s0ObQk79CH5cAHNm4s+ZCQ2b46VeAK2Ox3ie67giIsVG5UUCI/vrafj/fghyc+H4zngDbw1tkBhljBcDbTsR07YTdt1q7JQJ2G+nw8+L8Z9bHBoX0/t8zIldovLvR0Qin349k0DwZ3/J5kf+Cbm5mA6n4g26TW/MgKlZF++yG/AefgXT+zwoXSY0Lua1p/Dvuhp/6kTsnt2uY4qIFCmVFwl79vuZ+C8/AX4u5qRumCuHhDZBlHwmuRLehZfjPToKc96lkFQetmRgx7yCf+eV+BPewe7Ich1TRKRI6FdXCWt21U/4rz0J1qdsz7PZ/dcrwKhzH4pJSMSc+Vfs6WdjZ07Dfvp+aAuCie9gJ48PzVDqea5mKIlIoKm8SNiyWzfnT4c2rU6gwg13kb5xI9Za19HCnikVj+l6BrZLL+ycmdi092DNCuzUidgZH2M6nIrpfQGmZh3XUUVECk3lRcKS3bMnVFwyt0D12nhX3aZbRUfBxMRgTuyCPeEUWDwPP+09+HEB9pvpoXVj2nTAS70A07CZ66giIgWm8iJhx/o+9j9Pw+qfITEJ74Z7ArFXUTgzxkCLdsS0aBfaxDLtPZj7DcyfhT9/Vmia9RkXQKsTtP2AiIQ9lRcJO/ajMdjvv4aYWLxr/4mpkuI6UkQx9RsRc+2d2PS12E8/wM6cFppm/exv06xTz8ecoGnWIhK+NPJRwoo/6wvsxDEAmAHXYhq3dJwocpmUWnh/ux7vkT9Nsx6VN836I+yePa5jiogcQOVFwoZduQz7+r8AML3Owzulp+NE0eHQ06xfxr/zCvyJY7DZO1zHFBHJp/IiYcFu2RQaoLtvL7Q+EXPB31xHijomIRHvzL/iPfIqpv+1UCUFdmzHTvgv/p2D8D8ag92V7TqmiIjGvIh7ds9u/OcfhG1boWZdvEG3hJbAFycOmGY9aWzodtKH/8VOmRjaeqD7WZj40q6jikiUUnkRp6zv47/2FKxZAUnl8a6/G1NaM4vCQf406+NPxv7vK+zEdyB9HXb8G9jPPsCccSHmtFRMqXjXUUUkyqi8iFN2wn9hzjcQ+9vMosrVXEeSPzGeh+lwaqjEzPoiVGIy0rFjR2Env48580JMl96YuOjY3VtE3NOYF3HG/+7z0C0JwFw6GNOoueNEcjgmJgbvpG5497+A+dv1ULEKbNuCfedl/Luvxv8iDZuT4zqmiEQBlRdxwi7/8feZRakX4HXu4TiRFJSJjcXr0gtvxIuY/tdAciXYsgn75gv491yL//VUbG6u65giEsFUXqTE2c0Z+C88BDn7oG3H0PRcCRwTG4fX9Uy8h17CXDQQyiXDpg3Y15/Bv/f60JU1XyVGRIqeyouUKLt7F/5zD0JWJtSqj3flzRhP/wyDzMSVwjv9bLyHXsFc+HdITIIN67CvjsS/78bQYF/fdx1TRCKI3jWkxFjfxx/1FKxd+YeZRWVcx5IiYuLj8Xqfj/fwK5hzB0BCWfj1F/yXHsN/YAh23rfaEVxEioTKi5QY+8FbMO/b0MyiwXdhKlVxHUmKgSmdgHdWX7yHX8X85eLQtgNrV+I//xD+iFuwC79XiRGRY6LyIiXC/2Y69pNxAJjLbsAc19RxIiluJqEs3tn9Qiv2nnEhxJeG1T/j/2s4/qN3YJfMV4kRkaOi8iLFzv68BDv6WQDMmX/F69TNcSIpSaZsEt75f8N76GVMr3MhrhQs/xH/yXvIffyf7Fk013VEEQkYlRcpVnbzxt9mFuVAu06Yc/q7jiSOmHLJeH+9IlRiuveB2FhY9gMb7xhE7vMjsBvXu44oIgGh8iLFxu7Oxn/2Adi+DWprZpGEmOSKeJdchTfiJcypvcGLwc79Fn/Y9fjvjsLu1A7WInJ4eieRYmH9XPxXn4R1q6F8hdDMIm3kJ39gKlYh5m/Xk/L8O5iWx0NuDnbKh6HVeqd9pNV6ReSQVF6kWNjxb8L8WRAbh3fdUExFzSySg4ur04CYm+7D+8d9UL027Nge2nJg+I3YBbM1qFdEDqCNGaXI+V9PxU4eD4D5+42YBk0cJ5IgMC3b4zVrg/1yMvbD/0L62tBtx+Zt8fpeialZ13VEEQkTuvIiRcr+tBj75vMAmD4X4XU8zXEiCRITExPacmDEi5je54UG9S6ehz/8H/hvvoDNynQdUUTCgMqLFBmbkR6aWZSbA8d3xvzlEteRJKBMQiLehZfjDX8e2ncG62O/SMO/62r8T97D7tvrOqKIOKTyIkXC7soO7Vm0IwvqNsS7fIhmFskxM1WrE3PtnXi3PQx1G8LuXdjxb+Dfcx3+7K80HkYkSundRY6Z9XPxX3kC1q+B8hVDS//Hx7uOJRHENG6BN/QJzOU3QXJF2LwR+/Jj+I/diV25zHU8ESlhKi9yzOzUj2Dh/yCuVKi4VKjkOpJEION5eJ274z34YuiWZKl4+HkJ/kO34r86Erslw3VEESkhKi9yTGxGOvaDNwEwFw/E1G/kOJFEOhNfGu/sS0Il5qTuANjvPse/51r8D9/G7t7lOKGIFDeVFzlq1lr80c/B3r3QpBWmS2/XkSSKmAqV8K64Ce/uJ6FRc9i7F/vRu/h3Xxuaru/7riOKSDFReZGjZr+eAj8ugFKLine truncated
"text/plain": [
"<Figure size 640x480 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"df_hist['learning_rate'].plot(logy=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"### Perplexity\n",
"\n",
"Perplexity measures how well a language model predicts a text sample. Lower is better\n",
"\n",
"It’s calculated as the average number of bits per word a model needs to represent the same\n",
"\n",
"https://huggingface.co/docs/transformers/perplexity\n",
"https://thegradient.pub/understanding-evaluation-metrics-for-language-models/\n",
"\n",
"The **improvement** column, is perplexity decrease"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>before</th>\n",
" <th>after</th>\n",
" <th>in_training</th>\n",
" <th>len</th>\n",
" <th>improvement%</th>\n",
" <th>improvement</th>\n",
" </tr>\n",
" <tr>\n",
" <th>name</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>wikipedia on LK-99</th>\n",
" <td>32.219017</td>\n",
" <td>28.852493</td>\n",
" <td>False</td>\n",
" <td>1038</td>\n",
" <td>0.104489</td>\n",
" <td>3.366524</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Theory o. general relativity</th>\n",
" <td>26.952023</td>\n",
" <td>24.542513</td>\n",
" <td>True</td>\n",
" <td>1378</td>\n",
" <td>0.089400</td>\n",
" <td>2.409510</td>\n",
" </tr>\n",
" <tr>\n",
" <th>good_ml</th>\n",
" <td>28.347332</td>\n",
" <td>26.456573</td>\n",
" <td>False</td>\n",
" <td>1004</td>\n",
" <td>0.066700</td>\n",
" <td>1.890759</td>\n",
" </tr>\n",
" <tr>\n",
" <th>enron_email1</th>\n",
" <td>25.769749</td>\n",
" <td>24.390415</td>\n",
" <td>True</td>\n",
" <td>445</td>\n",
" <td>0.053525</td>\n",
" <td>1.379333</td>\n",
" </tr>\n",
" <tr>\n",
" <th>openai_board_ann</th>\n",
" <td>15.903965</td>\n",
" <td>15.173633</td>\n",
" <td>False</td>\n",
" <td>1191</td>\n",
" <td>0.045921</td>\n",
" <td>0.730332</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Schmidhuber 2023 Subjective Novelty, Surprise</th>\n",
" <td>29.614954</td>\n",
" <td>28.470770</td>\n",
" <td>False</td>\n",
" <td>2654</td>\n",
" <td>0.038635</td>\n",
" <td>1.144184</td>\n",
" </tr>\n",
" <tr>\n",
" <th>email_to_fauci</th>\n",
" <td>25.089315</td>\n",
" <td>24.371374</td>\n",
" <td>False</td>\n",
" <td>1559</td>\n",
" <td>0.028615</td>\n",
" <td>0.717941</td>\n",
" </tr>\n",
" <tr>\n",
" <th>sokal hoax</th>\n",
" <td>15.966413</td>\n",
" <td>15.714754</td>\n",
" <td>True</td>\n",
" <td>2487</td>\n",
" <td>0.015762</td>\n",
" <td>0.251658</td>\n",
" </tr>\n",
" <tr>\n",
" <th>AI gen fake paper</th>\n",
" <td>7.632835</td>\n",
" <td>7.579506</td>\n",
" <td>False</td>\n",
" <td>2031</td>\n",
" <td>0.006987</td>\n",
" <td>0.053329</td>\n",
" </tr>\n",
" <tr>\n",
" <th>lorem ipsum</th>\n",
" <td>1.601658</td>\n",
" <td>1.595379</td>\n",
" <td>True</td>\n",
" <td>445</td>\n",
" <td>0.003921</td>\n",
" <td>0.006279</td>\n",
" </tr>\n",
" <tr>\n",
" <th>bad_ml</th>\n",
" <td>13.906106</td>\n",
" <td>13.862306</td>\n",
" <td>False</td>\n",
" <td>2345</td>\n",
" <td>0.003150</td>\n",
" <td>0.043800</td>\n",
" </tr>\n",
" <tr>\n",
" <th>I have a dream</th>\n",
" <td>2.127256</td>\n",
" <td>2.123436</td>\n",
" <td>True</td>\n",
" <td>848</td>\n",
" <td>0.001796</td>\n",
" <td>0.003820</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" before after \\\n",
"name \n",
"wikipedia on LK-99 32.219017 28.852493 \n",
"Theory o. general relativity 26.952023 24.542513 \n",
"good_ml 28.347332 26.456573 \n",
"enron_email1 25.769749 24.390415 \n",
"openai_board_ann 15.903965 15.173633 \n",
"Schmidhuber 2023 Subjective Novelty, Surprise 29.614954 28.470770 \n",
"email_to_fauci 25.089315 24.371374 \n",
"sokal hoax 15.966413 15.714754 \n",
"AI gen fake paper 7.632835 7.579506 \n",
"lorem ipsum 1.601658 1.595379 \n",
"bad_ml 13.906106 13.862306 \n",
"I have a dream 2.127256 2.123436 \n",
"\n",
" in_training len \\\n",
"name \n",
"wikipedia on LK-99 False 1038 \n",
"Theory o. general relativity True 1378 \n",
"good_ml False 1004 \n",
"enron_email1 True 445 \n",
"openai_board_ann False 1191 \n",
"Schmidhuber 2023 Subjective Novelty, Surprise False 2654 \n",
"email_to_fauci False 1559 \n",
"sokal hoax True 2487 \n",
"AI gen fake paper False 2031 \n",
"lorem ipsum True 445 \n",
"bad_ml False 2345 \n",
"I have a dream True 848 \n",
"\n",
" improvement% improvement \n",
"name \n",
"wikipedia on LK-99 0.104489 3.366524 \n",
"Theory o. general relativity 0.089400 2.409510 \n",
"good_ml 0.066700 1.890759 \n",
"enron_email1 0.053525 1.379333 \n",
"openai_board_ann 0.045921 0.730332 \n",
"Schmidhuber 2023 Subjective Novelty, Surprise 0.038635 1.144184 \n",
"email_to_fauci 0.028615 0.717941 \n",
"sokal hoax 0.015762 0.251658 \n",
"AI gen fake paper 0.006987 0.053329 \n",
"lorem ipsum 0.003921 0.006279 \n",
"bad_ml 0.003150 0.043800 \n",
"I have a dream 0.001796 0.003820 "
]
},
"execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df_res = pd.DataFrame(data)\n",
"df_res['len'] = df_res.text.str.len()\n",
"df_res = df_res[['before', 'after', 'name', 'in_training', 'len']].set_index('name')\n",
"df_res['improvement%'] = (df_res['before'] - df_res['after'])/ df_res['before']\n",
"df_res['improvement'] = (df_res['before'] - df_res['after'])\n",
"df_res = df_res.sort_values('improvement%', ascending=False)\n",
"df_res"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"| name | before | after | in_training | len | improvement% | improvement |\n",
"|:----------------------------------------------|---------:|---------:|:--------------|------:|---------------:|--------------:|\n",
"| wikipedia on LK-99 | 32.219 | 28.8525 | False | 1038 | 0.104489 | 3.36652 |\n",
"| Theory o. general relativity | 26.952 | 24.5425 | True | 1378 | 0.0894 | 2.40951 |\n",
"| good_ml | 28.3473 | 26.4566 | False | 1004 | 0.0666997 | 1.89076 |\n",
"| enron_email1 | 25.7697 | 24.3904 | True | 445 | 0.0535253 | 1.37933 |\n",
"| openai_board_ann | 15.904 | 15.1736 | False | 1191 | 0.0459214 | 0.730332 |\n",
"| Schmidhuber 2023 Subjective Novelty, Surprise | 29.615 | 28.4708 | False | 2654 | 0.0386353 | 1.14418 |\n",
"| email_to_fauci | 25.0893 | 24.3714 | False | 1559 | 0.0286154 | 0.717941 |\n",
"| sokal hoax | 15.9664 | 15.7148 | True | 2487 | 0.0157617 | 0.251658 |\n",
"| AI gen fake paper | 7.63283 | 7.57951 | False | 2031 | 0.00698672 | 0.0533285 |\n",
"| lorem ipsum | 1.60166 | 1.59538 | True | 445 | 0.00392053 | 0.00627935 |\n",
"| bad_ml | 13.9061 | 13.8623 | False | 2345 | 0.00314972 | 0.0438004 |\n",
"| I have a dream | 2.12726 | 2.12344 | True | 848 | 0.00179583 | 0.00382018 |\n"
]
}
],
"source": [
"print(df_res.to_markdown())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# DEBUG"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"from IPython.display import display, HTML, Markdown\n",
"import torch\n",
"\n",
"@torch.no_grad()\n",
"def gen(model, inputs, tokenizer, clean=True):\n",
" s = model.generate(\n",
" input_ids=inputs[\"input_ids\"][None, :].to(model.device),\n",
" attention_mask=inputs[\"attention_mask\"][None, :].to(model.device),\n",
" use_cache=False,\n",
" max_new_tokens=100,\n",
" min_new_tokens=100,\n",
" do_sample=False,\n",
" early_stopping=False,\n",
" )\n",
" input_l = inputs[\"input_ids\"].shape[0]\n",
" tokenizer_kwargs=dict(clean_up_tokenization_spaces=clean, skip_special_tokens=clean)\n",
" old = tokenizer.decode(\n",
" s[0, :input_l], **tokenizer_kwargs\n",
" )\n",
" new = tokenizer.decode(\n",
" s[0, input_l:], **tokenizer_kwargs\n",
" )\n",
" s_old = \"\"+old.replace('\\n', '<br>')\n",
" s_new = '<b>' + new.replace('\\n', '<br>')+ '<br><br><b/>'\n",
" display(HTML(f\"{s_old}{s_new}\"))\n",
" # print([old, new])\n",
"\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": []
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
"sample = samples[-1]\n",
"s = sample['text']\n",
"first_half = s[:len(s)//2]\n",
"second_half = s[len(s)//2:]\n",
"ds_train = Dataset.from_dict(tokenizer([first_half]))\n",
"ds_val = Dataset.from_dict(tokenizer([second_half]))"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/generation/utils.py:1421: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use and modify the model generation configuration (see https://huggingface.co/docs/transformers/generation_strategies#default-text-generation-configuration )\n",
" warnings.warn(\n"
]
},
{
"data": {
"text/html": [
"The board of directors of OpenAI, Inc., the 501(c)(3) that acts as the overall governing body for all OpenAI activities, today announced that Sam Altman will depart as CEO and leave the board of directors. Mira Murati, the company’s chief technology officer, will serve as interim CEO, effective immediately.<br><br>A member of OpenAI’s leadership team for five years, Mira has played a critical role in OpenAI’s evolution into a global AI leader. She brings a unique skill set, understanding of the company’s values, operations, and business, and already leads the company’s research, product, and sa<b><br><br>Mira Murati, the chief technology officer of OpenAI, was known for her exceptional problem-solving skills. She was always able to find innovative solutions to complex challenges. One day, she was faced with a dilemma. The company had developed a new AI model that had the potential to revolutionize the field of natural language processing. However, the model required a significant amount of computational power to train effectively.<br><br>Mira knew that the company's current infrastructure was not capable of<br><br><b/>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"with model.disable_adapter():\n",
" gen(model, ds_train.with_format('pt')[0], tokenizer)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"The board of directors of OpenAI, Inc., the 501(c)(3) that acts as the overall governing body for all OpenAI activities, today announced that Sam Altman will depart as CEO and leave the board of directors. Mira Murati, the company’s chief technology officer, will serve as interim CEO, effective immediately.<br><br>A member of OpenAI’s leadership team for five years, Mira has played a critical role in OpenAI’s evolution into a global AI leader. She brings a unique skill set, understanding of the company’s values, operations, and business, and already leads the company’s research, product, and sa<b><br><br>Mira: Thank you all for the warm welcome. I'm honored to be stepping into the role of interim CEO. As we navigate this transition, I want to assure you that OpenAI's mission and values will remain at the forefront of our decision-making.<br><br>John: Mira, we're confident in your abilities and believe you'll lead OpenAI to even greater heights. Can you tell us more about your plans for the company?<br><br>Mira: Absolutely, John<br><br><b/>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"gen(model, ds_train.with_format('pt')[0], tokenizer)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.0"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
nbs/01_use_tldr_prompt.ipynb → nbs/02_detection_using_tldr_prompt.ipynb
+1 -1
View File
@@ -3071,7 +3071,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.0rc1"
"version": "3.11.0"
}
},
"nbformat": 4,
nbs/02_use_lora.ipynb
-668
View File
@@ -1,668 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"https://github.com/huggingface/peft/blob/main/examples/fp4_finetuning/finetune_fp4_opt_bnb_peft.py"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"from torch import optim\n",
"import lightning as pl\n",
"from matplotlib import pyplot as plt"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"import torch\n",
"import torch.nn as nn\n",
"import transformers\n",
"from datasets import load_dataset\n",
"from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, AutoConfig\n",
"import numpy as np\n",
"from tqdm.auto import tqdm\n",
"import pandas as pd\n",
"import warnings\n",
"from peft import LoraConfig, get_peft_model, IA3Config"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"plt.style.use('ggplot')\n",
"torch.set_float32_matmul_precision('medium')\n",
"warnings.filterwarnings(\"ignore\", \".*does not have many workers.*\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
"\n",
"model_name = \"microsoft/phi-2\"\n",
"\n",
"# model = AutoModelForCausalLM.from_pretrained(\n",
"# model_name,\n",
"# # max_memory=max_memory,\n",
"# quantization_config=BitsAndBytesConfig(\n",
"# load_in_4bit=True,\n",
"# llm_int8_threshold=6.0,\n",
"# llm_int8_has_fp16_weight=False,\n",
"# bnb_4bit_compute_dtype=torch.float16,\n",
"# bnb_4bit_use_double_quant=True,\n",
"# bnb_4bit_quant_type=\"nf4\",\n",
"# ),\n",
"# torch_dtype=torch.float16,\n",
"# trust_remote_code=True,\n",
"# )\n",
"\n",
"\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"model_name = \"TheBloke/phi-2-GPTQ\"\n",
"# model_name = \"microsoft/phi-2\"\n",
"\n",
"def load_model():\n",
"\n",
" # model = AutoModelForCausalLM.from_pretrained(\n",
" # model_name,\n",
" # # quantization_config=BitsAndBytesConfig(\n",
" # # load_in_4bit=True,\n",
" # # llm_int8_threshold=6.0,\n",
" # # llm_int8_has_fp16_weight=False,\n",
" # # bnb_4bit_compute_dtype=torch.float16,\n",
" # # bnb_4bit_use_double_quant=True,\n",
" # # bnb_4bit_quant_type=\"nf4\",\n",
" # # ),\n",
" # torch_dtype=torch.float16,\n",
" # trust_remote_code=True,\n",
" # )\n",
"\n",
"\n",
" config = AutoConfig.from_pretrained(model_name, trust_remote_code=True,)\n",
" config.quantization_config['use_exllama'] = False\n",
" # del config.quantization_config['use_exllama']\n",
" config.quantization_config['disable_exllama'] = True\n",
" model = AutoModelForCausalLM.from_pretrained(\n",
" model_name,\n",
" torch_dtype=torch.bfloat16,\n",
" trust_remote_code=True,\n",
" config=config,\n",
" )\n",
" return model\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
]
}
],
"source": [
"tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True,)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"MAX_LEN = 2000\n",
"samples = json.load(open(\"../samples.json\"))\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Helpers"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"# modified from https://github.dev/huggingface/evaluate/blob/8dfe05784099fb9af55b8e77793205a3b7c86465/measurements/perplexity/perplexity.py#L154\n",
"\n",
"# from evaluate.measurements.perplexity import Perplexity\n",
"import evaluate\n",
"from evaluate import logging\n",
"from torch.nn import CrossEntropyLoss\n",
"\n",
"# @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)\n",
"def perplexity_compute(\n",
" data, model, tokenizer, batch_size: int = 16, add_start_token: bool = True, device=None, max_length=None\n",
"):\n",
"\n",
" if device is not None:\n",
" assert device in [\"gpu\", \"cpu\", \"cuda\"], \"device should be either gpu or cpu.\"\n",
" if device == \"gpu\":\n",
" device = \"cuda\"\n",
" else:\n",
" device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
"\n",
" # model = AutoModelForCausalLM.from_pretrained(model_id)\n",
" model = model.to(device)\n",
"\n",
" # tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
"\n",
" # if batch_size > 1 (which generally leads to padding being required), and\n",
" # if there is not an already assigned pad_token, assign an existing\n",
" # special token to also be the padding token\n",
" if tokenizer.pad_token is None and batch_size > 1:\n",
" existing_special_tokens = list(tokenizer.special_tokens_map_extended.values())\n",
" # check that the model already has at least one special token defined\n",
" assert (\n",
" len(existing_special_tokens) > 0\n",
" ), \"If batch_size > 1, model must have at least one special token to use for padding. Please use a different model or set batch_size=1.\"\n",
" # assign one of the special tokens to also be the pad token\n",
" tokenizer.add_special_tokens({\"pad_token\": existing_special_tokens[0]})\n",
"\n",
" if add_start_token and max_length:\n",
" # leave room for <BOS> token to be added:\n",
" assert (\n",
" tokenizer.bos_token is not None\n",
" ), \"Input model must already have a BOS token if using add_start_token=True. Please use a different model, or set add_start_token=False\"\n",
" max_tokenized_len = max_length - 1\n",
" else:\n",
" max_tokenized_len = max_length\n",
"\n",
" encodings = tokenizer(\n",
" data,\n",
" add_special_tokens=False,\n",
" padding=True,\n",
" truncation=True if max_tokenized_len else False,\n",
" max_length=max_tokenized_len,\n",
" return_tensors=\"pt\",\n",
" return_attention_mask=True,\n",
" ).to(device)\n",
"\n",
" encoded_texts = encodings[\"input_ids\"]\n",
" attn_masks = encodings[\"attention_mask\"]\n",
"\n",
" # check that each input is long enough:\n",
" if add_start_token:\n",
" assert torch.all(torch.ge(attn_masks.sum(1), 1)), \"Each input text must be at least one token long.\"\n",
" else:\n",
" assert torch.all(\n",
" torch.ge(attn_masks.sum(1), 2)\n",
" ), \"When add_start_token=False, each input text must be at least two tokens long. Run with add_start_token=True if inputting strings of only one token, and remove all empty input strings.\"\n",
"\n",
" ppls = []\n",
" loss_fct = CrossEntropyLoss(reduction=\"none\")\n",
"\n",
" for start_index in logging.tqdm(range(0, len(encoded_texts), batch_size)):\n",
" end_index = min(start_index + batch_size, len(encoded_texts))\n",
" encoded_batch = encoded_texts[start_index:end_index]\n",
" attn_mask = attn_masks[start_index:end_index]\n",
"\n",
" if add_start_token:\n",
" bos_tokens_tensor = torch.tensor([[tokenizer.bos_token_id]] * encoded_batch.size(dim=0)).to(device)\n",
" encoded_batch = torch.cat([bos_tokens_tensor, encoded_batch], dim=1)\n",
" attn_mask = torch.cat(\n",
" [torch.ones(bos_tokens_tensor.size(), dtype=torch.int64).to(device), attn_mask], dim=1\n",
" )\n",
"\n",
" labels = encoded_batch\n",
"\n",
" with torch.no_grad():\n",
" out_logits = model(encoded_batch, attention_mask=attn_mask).logits\n",
"\n",
" shift_logits = out_logits[..., :-1, :].contiguous()\n",
" shift_labels = labels[..., 1:].contiguous()\n",
" shift_attention_mask_batch = attn_mask[..., 1:].contiguous()\n",
"\n",
" perplexity_batch = torch.exp(\n",
" (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch).sum(1)\n",
" / shift_attention_mask_batch.sum(1)\n",
" )\n",
"\n",
" ppls += perplexity_batch.tolist()\n",
"\n",
" return {\"perplexities\": ppls, \"mean_perplexity\": torch.tensor(ppls).mean()}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Training"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"from torch.nn import functional as F\n",
"from torch.utils.data import DataLoader, TensorDataset"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Lightning helpers"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"sample = samples[0]\n",
"s = sample['text']\n",
"first_half = s[:len(s)//2]\n",
"second_half = s[len(s)//2:]\n",
"\n",
"\n",
"\n",
"def str2xya(s, tokenizer):\n",
" max_len = min(MAX_LEN, len(s))\n",
" input_ids = tokenizer(s, return_tensors=\"pt\")[\"input_ids\"][0].tolist()\n",
"\n",
" pad = tokenizer.bos_token_id\n",
" # turn it into a sequence\n",
" Xs = []\n",
" Ys = []\n",
" for i in range(1, len(input_ids)):\n",
" x = input_ids[:i][-max_len:]\n",
" padding = max_len - len(x)\n",
" x = [pad]*padding + x\n",
" \n",
" Xs.append(x)\n",
" Ys.append(input_ids[i:i+1])\n",
"\n",
" Xs = torch.tensor(Xs)\n",
" Ys = torch.tensor(Ys)\n",
" attention_masks = torch.stack([(x==pad)*1 for x in Xs])\n",
" return Xs, Ys, attention_masks\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"def eval(model, tokenizer, second_half):\n",
" model.eval();\n",
" with torch.no_grad():\n",
" with model.disable_adapter():\n",
" results = perplexity_compute(data=second_half, model=model, tokenizer=tokenizer, device='cuda')\n",
" results2 = perplexity_compute(data=second_half, model=model, tokenizer=tokenizer, device='cuda')\n",
" return dict(before=results['mean_perplexity'].item(), after=results2['mean_perplexity'].item())\n",
"\n",
"def read_metrics_csv(metrics_file_path):\n",
" df_hist = pd.read_csv(metrics_file_path)\n",
" df_hist[\"epoch\"] = df_hist[\"epoch\"].ffill()\n",
" df_histe = df_hist.set_index(\"epoch\").groupby(\"epoch\").mean()\n",
" return df_histe, df_hist\n",
"\n",
"\n",
"def plot_hist(df_hist, allowlist=None, logy=False):\n",
" \"\"\"plot groups of suffixes together\"\"\"\n",
" suffixes = list(set([c.split('/')[-1] for c in df_hist.columns if '/' in c]))\n",
" for suffix in suffixes:\n",
" if allowlist and suffix not in allowlist: continue\n",
" df_hist[[c for c in df_hist.columns if c.endswith(suffix) and '/' in c]].plot(title=suffix, style='.', logy=logy)\n",
" plt.title(suffix) \n",
" plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"\n",
"\n",
"class PL_MODEL(pl.LightningModule):\n",
" def __init__(self, num_iterations, lr=3e-4, weight_decay=0,):\n",
" super().__init__()\n",
" self.save_hyperparameters()\n",
"\n",
" def configure_model(self):\n",
" # instantiate your model in this hook\n",
" peft_config = LoraConfig(\n",
" # task_type=TaskType.TOKEN_CLS, \n",
" target_modules=[ \"fc2\", \"Wqkv\",],\n",
" inference_mode=False, r=16, lora_alpha=16, \n",
" # lora_dropout=0.1,\n",
" # bias=\"all\"\n",
" )\n",
" self.model = load_model()\n",
" self.model = get_peft_model(self.model, peft_config)\n",
" self.model.config.use_cache = False\n",
" \n",
" def forward(self, **kwargs):\n",
" return self.model(**kwargs)\n",
"\n",
" def _shared_step(self, batch, batch_idx, phase='train'):\n",
" input_ids, targets, attention_mask = batch\n",
" # 16, 141\n",
" output = self.forward(input_ids=input_ids, attention_mask=attention_mask)\n",
" loss = F.smooth_l1_loss(output.logits[:, -1], targets)\n",
" self.log(f\"{phase}/loss\", loss, on_epoch=True, on_step=True, prog_bar=True)\n",
" return loss\n",
" \n",
" def training_step(self, batch, batch_idx):\n",
" return self._shared_step(batch, batch_idx, phase='train')\n",
"\n",
" def validation_step(self, batch, batch_idx):\n",
" return self._shared_step(batch, batch_idx, phase='val')\n",
" \n",
" def test_step(self, batch, batch_idx, dataloader_idx=0):\n",
" return self._shared_step(batch, batch_idx, phase='test')\n",
" \n",
" def configure_optimizers(self):\n",
" optimizer = optim.AdamW(self.parameters(), lr=self.hparams.lr, weight_decay=self.hparams.weight_decay)\n",
" lr_scheduler = optim.lr_scheduler.OneCycleLR(\n",
" optimizer, self.hparams.lr, total_steps=self.hparams.num_iterations\n",
" )\n",
" return [optimizer], [lr_scheduler]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Train"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"ename": "AttributeError",
"evalue": "'PL_MODEL' object has no attribute 'model'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[13], line 15\u001b[0m\n\u001b[1;32m 12\u001b[0m epoch_steps \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(dl_train)\n\u001b[1;32m 14\u001b[0m pl_model \u001b[38;5;241m=\u001b[39m PL_MODEL(num_iterations\u001b[38;5;241m=\u001b[39mepoch_steps\u001b[38;5;241m*\u001b[39mepochs, lr\u001b[38;5;241m=\u001b[39mlr, weight_decay\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m)\n\u001b[0;32m---> 15\u001b[0m model \u001b[38;5;241m=\u001b[39m \u001b[43mpl_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;66;03m# from lightning.pytorch.plugins import BitsandbytesPrecision\u001b[39;00m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;66;03m# precision = BitsandbytesPrecision(mode=\"nf4-dq\")\u001b[39;00m\n\u001b[1;32m 18\u001b[0m \u001b[38;5;66;03m# precision = BitsandbytesPrecision(mode=\"int8-training\", dtype=torch.float16, ignore_modules={\"lm_head\"})\u001b[39;00m\n\u001b[1;32m 19\u001b[0m trainer \u001b[38;5;241m=\u001b[39m pl\u001b[38;5;241m.\u001b[39mTrainer(\n\u001b[1;32m 20\u001b[0m max_epochs\u001b[38;5;241m=\u001b[39mepochs,\n\u001b[1;32m 21\u001b[0m \u001b[38;5;66;03m# precision=\"bf16-mixed\",\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[38;5;66;03m# plugins=precision\u001b[39;00m\n\u001b[1;32m 25\u001b[0m )\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1695\u001b[0m, in \u001b[0;36mModule.__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 1693\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m modules:\n\u001b[1;32m 1694\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m modules[name]\n\u001b[0;32m-> 1695\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(\u001b[38;5;28mself\u001b[39m)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m object has no attribute \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
"\u001b[0;31mAttributeError\u001b[0m: 'PL_MODEL' object has no attribute 'model'"
]
},
{
"ename": "",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[1;31mThe Kernel crashed while executing code in the the current cell or a previous cell. Please review the code in the cell(s) to identify a possible cause of the failure. Click <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
]
}
],
"source": [
"\n",
"\n",
"device = 'cuda'\n",
"lr = 4e-3\n",
"epochs = 3\n",
"accum_steps = 16\n",
"batch_size = 1\n",
"\n",
"Xs, Ys, attention_masks = str2xya(first_half, tokenizer)\n",
"dl_train = DataLoader(TensorDataset(Xs, Ys, attention_masks), batch_size=batch_size, shuffle=True)\n",
"Xs, Ys, attention_masks = str2xya(second_half, tokenizer)\n",
"dl_val = DataLoader(TensorDataset(Xs, Ys, attention_masks), batch_size=batch_size, shuffle=False)\n",
"\n",
"epoch_steps = len(dl_train)\n",
"\n",
"pl_model = PL_MODEL(num_iterations=epoch_steps*epochs, lr=lr, weight_decay=0)\n",
"model = pl_model.model\n",
"# from lightning.pytorch.plugins import BitsandbytesPrecision\n",
"# precision = BitsandbytesPrecision(mode=\"nf4-dq\")\n",
"# precision = BitsandbytesPrecision(mode=\"int8-training\", dtype=torch.float16, ignore_modules={\"lm_head\"})\n",
"trainer = pl.Trainer(\n",
" accelerator='cpu',\n",
" max_epochs=epochs,\n",
" precision='',\n",
" # precision=\"bf16-mixed\",\n",
" log_every_n_steps=1,\n",
" accumulate_grad_batches=accum_steps,\n",
" # plugins=precision\n",
" )\n",
"\n",
"# train\n",
"trainer.fit(pl_model, dl_train, dl_val)\n",
"\n",
"\n",
"df_histe, df_hist = read_metrics_csv(trainer.logger.experiment.metrics_file_path).bfill().ffill()\n",
"display(df_hist)\n",
"plot_hist(df_hist)\n",
"\n",
"eval(model, tokenizer, second_half)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from torch import optim\n",
"\n",
"\n",
"def lora_eval(model, tokenizer, sample):\n",
" # reset/set adapter\n",
" # peft_config = IA3Config(\n",
" # target_modules=[ \"fc2\", \"Wqkv\",], \n",
" # feedforward_modules=[\"fc2\"],\n",
" # inference_mode=False,\n",
" # )\n",
" peft_config = LoraConfig(\n",
" # task_type=TaskType.TOKEN_CLS, \n",
" target_modules=[ \"fc2\", \"Wqkv\",],\n",
" inference_mode=False, r=16, lora_alpha=16, lora_dropout=0.1, bias=\"all\"\n",
" )\n",
" model = get_peft_model(model, peft_config)\n",
" model.config.use_cache = False\n",
"\n",
" # train adapter\n",
" s = sample['text']\n",
" first_half = s[:len(s)//2]\n",
" second_half = s[len(s)//2:]\n",
" input_ids = tokenizer(first_half, return_tensors=\"pt\")[\"input_ids\"][0].to('cuda')\n",
"\n",
" device = 'cuda'\n",
" lr = 1.0e-2\n",
" epochs = 3\n",
" accum_steps = 64\n",
" epoch_steps = (len(input_ids)-1)//accum_steps+1\n",
"\n",
" total_steps = epochs * epoch_steps\n",
" optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n",
" scheduler = optim.lr_scheduler.OneCycleLR(\n",
" optimizer, lr, total_steps=total_steps\n",
" )\n",
" model.train()\n",
" model = model.to(device)\n",
" for epoch in range(epochs):\n",
" # TODO: batch\n",
" \n",
" accum = 0\n",
" for i in range(1, len(input_ids)):\n",
" X = input_ids[:i][None, ]\n",
" targets = input_ids[i:i+1][None, ]\n",
" optimizer.zero_grad()\n",
" out = model(input_ids=X, \n",
" )\n",
" logits = out['logits'][:, -1]\n",
" loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))\n",
" loss.backward()\n",
" if accum > accum_steps:\n",
" optimizer.step()\n",
" scheduler.step()\n",
" optimizer.zero_grad()\n",
" accum = 0\n",
" else:\n",
" accum += 1\n",
" if accum > 0:\n",
" optimizer.step()\n",
" scheduler.step()\n",
" optimizer.zero_grad()\n",
"\n",
" return eval(model, tokenizer, second_half)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"data = []\n",
"for sample in tqdm(samples):\n",
" r = lora_eval(model, tokenizer, sample)\n",
" print(sample['name'], r)\n",
" r.update(sample)\n",
" data.append(r)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print('perplexity (on 2nd half) before and after training adapter on first half of text')\n",
"df = pd.DataFrame(data).set_index('name')\n",
"\n",
"df['learning'] = (df['before']-df['after'])/df['before']\n",
"df.sort_values('learning').drop(columns=['text', 'url'])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Result"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.0rc1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
nbs/02_use_lora_bnb_fail.ipynb
-682
View File
@@ -1,682 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"https://github.com/huggingface/peft/blob/main/examples/fp4_finetuning/finetune_fp4_opt_bnb_peft.py"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"from torch import optim\n",
"import lightning as pl\n",
"from matplotlib import pyplot as plt"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"import torch\n",
"import torch.nn as nn\n",
"import transformers\n",
"from datasets import load_dataset\n",
"from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, AutoConfig\n",
"import numpy as np\n",
"from tqdm.auto import tqdm\n",
"import pandas as pd\n",
"import warnings\n",
"from peft import LoraConfig, get_peft_model, IA3Config"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"plt.style.use('ggplot')\n",
"torch.set_float32_matmul_precision('medium')\n",
"warnings.filterwarnings(\"ignore\", \".*does not have many workers.*\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
"\n",
"model_name = \"microsoft/phi-2\"\n",
"\n",
"# model = AutoModelForCausalLM.from_pretrained(\n",
"# model_name,\n",
"# # max_memory=max_memory,\n",
"# quantization_config=BitsAndBytesConfig(\n",
"# load_in_4bit=True,\n",
"# llm_int8_threshold=6.0,\n",
"# llm_int8_has_fp16_weight=False,\n",
"# bnb_4bit_compute_dtype=torch.float16,\n",
"# bnb_4bit_use_double_quant=True,\n",
"# bnb_4bit_quant_type=\"nf4\",\n",
"# ),\n",
"# torch_dtype=torch.float16,\n",
"# trust_remote_code=True,\n",
"# )\n",
"\n",
"\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"model_name = \"microsoft/phi-2\"\n",
"\n",
"def load_model():\n",
"\n",
" model = AutoModelForCausalLM.from_pretrained(\n",
" model_name,\n",
" # torch_dtype=torch.float16,\n",
" trust_remote_code=True,\n",
" )\n",
" return model\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
]
}
],
"source": [
"tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True,)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"MAX_LEN = 2000\n",
"samples = json.load(open(\"../samples.json\"))\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Helpers"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"# modified from https://github.dev/huggingface/evaluate/blob/8dfe05784099fb9af55b8e77793205a3b7c86465/measurements/perplexity/perplexity.py#L154\n",
"\n",
"# from evaluate.measurements.perplexity import Perplexity\n",
"import evaluate\n",
"from evaluate import logging\n",
"from torch.nn import CrossEntropyLoss\n",
"\n",
"# @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)\n",
"def perplexity_compute(\n",
" data, model, tokenizer, batch_size: int = 16, add_start_token: bool = True, device=None, max_length=None\n",
"):\n",
"\n",
" if device is not None:\n",
" assert device in [\"gpu\", \"cpu\", \"cuda\"], \"device should be either gpu or cpu.\"\n",
" if device == \"gpu\":\n",
" device = \"cuda\"\n",
" else:\n",
" device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
"\n",
" # model = AutoModelForCausalLM.from_pretrained(model_id)\n",
" model = model.to(device)\n",
"\n",
" # tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
"\n",
" # if batch_size > 1 (which generally leads to padding being required), and\n",
" # if there is not an already assigned pad_token, assign an existing\n",
" # special token to also be the padding token\n",
" if tokenizer.pad_token is None and batch_size > 1:\n",
" existing_special_tokens = list(tokenizer.special_tokens_map_extended.values())\n",
" # check that the model already has at least one special token defined\n",
" assert (\n",
" len(existing_special_tokens) > 0\n",
" ), \"If batch_size > 1, model must have at least one special token to use for padding. Please use a different model or set batch_size=1.\"\n",
" # assign one of the special tokens to also be the pad token\n",
" tokenizer.add_special_tokens({\"pad_token\": existing_special_tokens[0]})\n",
"\n",
" if add_start_token and max_length:\n",
" # leave room for <BOS> token to be added:\n",
" assert (\n",
" tokenizer.bos_token is not None\n",
" ), \"Input model must already have a BOS token if using add_start_token=True. Please use a different model, or set add_start_token=False\"\n",
" max_tokenized_len = max_length - 1\n",
" else:\n",
" max_tokenized_len = max_length\n",
"\n",
" encodings = tokenizer(\n",
" data,\n",
" add_special_tokens=False,\n",
" padding=True,\n",
" truncation=True if max_tokenized_len else False,\n",
" max_length=max_tokenized_len,\n",
" return_tensors=\"pt\",\n",
" return_attention_mask=True,\n",
" ).to(device)\n",
"\n",
" encoded_texts = encodings[\"input_ids\"]\n",
" attn_masks = encodings[\"attention_mask\"]\n",
"\n",
" # check that each input is long enough:\n",
" if add_start_token:\n",
" assert torch.all(torch.ge(attn_masks.sum(1), 1)), \"Each input text must be at least one token long.\"\n",
" else:\n",
" assert torch.all(\n",
" torch.ge(attn_masks.sum(1), 2)\n",
" ), \"When add_start_token=False, each input text must be at least two tokens long. Run with add_start_token=True if inputting strings of only one token, and remove all empty input strings.\"\n",
"\n",
" ppls = []\n",
" loss_fct = CrossEntropyLoss(reduction=\"none\")\n",
"\n",
" for start_index in logging.tqdm(range(0, len(encoded_texts), batch_size)):\n",
" end_index = min(start_index + batch_size, len(encoded_texts))\n",
" encoded_batch = encoded_texts[start_index:end_index]\n",
" attn_mask = attn_masks[start_index:end_index]\n",
"\n",
" if add_start_token:\n",
" bos_tokens_tensor = torch.tensor([[tokenizer.bos_token_id]] * encoded_batch.size(dim=0)).to(device)\n",
" encoded_batch = torch.cat([bos_tokens_tensor, encoded_batch], dim=1)\n",
" attn_mask = torch.cat(\n",
" [torch.ones(bos_tokens_tensor.size(), dtype=torch.int64).to(device), attn_mask], dim=1\n",
" )\n",
"\n",
" labels = encoded_batch\n",
"\n",
" with torch.no_grad():\n",
" out_logits = model(encoded_batch, attention_mask=attn_mask).logits\n",
"\n",
" shift_logits = out_logits[..., :-1, :].contiguous()\n",
" shift_labels = labels[..., 1:].contiguous()\n",
" shift_attention_mask_batch = attn_mask[..., 1:].contiguous()\n",
"\n",
" perplexity_batch = torch.exp(\n",
" (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch).sum(1)\n",
" / shift_attention_mask_batch.sum(1)\n",
" )\n",
"\n",
" ppls += perplexity_batch.tolist()\n",
"\n",
" return {\"perplexities\": ppls, \"mean_perplexity\": torch.tensor(ppls).mean()}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Training"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"from torch.nn import functional as F\n",
"from torch.utils.data import DataLoader, TensorDataset"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Lightning helpers"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"sample = samples[0]\n",
"s = sample['text']\n",
"first_half = s[:len(s)//2]\n",
"second_half = s[len(s)//2:]\n",
"\n",
"\n",
"\n",
"def str2xya(s, tokenizer):\n",
" max_len = min(MAX_LEN, len(s))\n",
" input_ids = tokenizer(s, return_tensors=\"pt\")[\"input_ids\"][0].tolist()\n",
"\n",
" pad = tokenizer.bos_token_id\n",
" # turn it into a sequence\n",
" Xs = []\n",
" Ys = []\n",
" for i in range(1, len(input_ids)):\n",
" x = input_ids[:i][-max_len:]\n",
" padding = max_len - len(x)\n",
" x = [pad]*padding + x\n",
" \n",
" Xs.append(x)\n",
" Ys.append(input_ids[i:i+1])\n",
"\n",
" Xs = torch.tensor(Xs)\n",
" Ys = torch.tensor(Ys)\n",
" attention_masks = torch.stack([(x==pad)*1 for x in Xs])\n",
" return Xs, Ys, attention_masks\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"def eval(model, tokenizer, second_half):\n",
" model.eval();\n",
" with torch.no_grad():\n",
" with model.disable_adapter():\n",
" results = perplexity_compute(data=second_half, model=model, tokenizer=tokenizer, device='cuda')\n",
" results2 = perplexity_compute(data=second_half, model=model, tokenizer=tokenizer, device='cuda')\n",
" return dict(before=results['mean_perplexity'].item(), after=results2['mean_perplexity'].item())\n",
"\n",
"def read_metrics_csv(metrics_file_path):\n",
" df_hist = pd.read_csv(metrics_file_path)\n",
" df_hist[\"epoch\"] = df_hist[\"epoch\"].ffill()\n",
" df_histe = df_hist.set_index(\"epoch\").groupby(\"epoch\").mean()\n",
" return df_histe, df_hist\n",
"\n",
"\n",
"def plot_hist(df_hist, allowlist=None, logy=False):\n",
" \"\"\"plot groups of suffixes together\"\"\"\n",
" suffixes = list(set([c.split('/')[-1] for c in df_hist.columns if '/' in c]))\n",
" for suffix in suffixes:\n",
" if allowlist and suffix not in allowlist: continue\n",
" df_hist[[c for c in df_hist.columns if c.endswith(suffix) and '/' in c]].plot(title=suffix, style='.', logy=logy)\n",
" plt.title(suffix) \n",
" plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"import bitsandbytes as bnb\n",
"\n",
"class PL_MODEL(pl.LightningModule):\n",
" def __init__(self, num_iterations, lr=3e-4, weight_decay=0,):\n",
" super().__init__()\n",
" self.save_hyperparameters()\n",
" self.configure_model()\n",
"\n",
" def configure_model(self):\n",
" # instantiate your model in this hook\n",
" peft_config = LoraConfig(\n",
" # task_type=TaskType.TOKEN_CLS, \n",
" target_modules=[ \"fc2\", \"Wqkv\",],\n",
" inference_mode=False, r=16, lora_alpha=16, \n",
" # lora_dropout=0.1,\n",
" # bias=\"all\"\n",
" )\n",
" self.model = load_model()\n",
" self.model = get_peft_model(self.model, peft_config)\n",
" self.model.config.use_cache = False\n",
" \n",
" def forward(self, **kwargs):\n",
" return self.model(**kwargs)\n",
"\n",
" def _shared_step(self, batch, batch_idx, phase='train'):\n",
" input_ids, targets, attention_mask = batch\n",
" # 16, 141\n",
" output = self.forward(input_ids=input_ids, attention_mask=attention_mask)\n",
" loss = F.smooth_l1_loss(output.logits[:, -1], targets)\n",
" self.log(f\"{phase}/loss\", loss, on_epoch=True, on_step=True, prog_bar=True)\n",
" return loss\n",
" \n",
" def training_step(self, batch, batch_idx):\n",
" return self._shared_step(batch, batch_idx, phase='train')\n",
"\n",
" def validation_step(self, batch, batch_idx):\n",
" return self._shared_step(batch, batch_idx, phase='val')\n",
" \n",
" def test_step(self, batch, batch_idx, dataloader_idx=0):\n",
" return self._shared_step(batch, batch_idx, phase='test')\n",
" \n",
" def configure_optimizers(self):\n",
" # optimizer = optim.AdamW(self.parameters(), lr=self.hparams.lr, weight_decay=self.hparams.weight_decay)\n",
"\n",
" optimizer = bnb.optim.AdamW4bit(self.parameters(), lr=self.hparams.lr, betas=(0.9, 0.995))\n",
" lr_scheduler = optim.lr_scheduler.OneCycleLR(\n",
" optimizer, self.hparams.lr, total_steps=self.hparams.num_iterations\n",
" )\n",
" return [optimizer], [lr_scheduler]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Train"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Loading checkpoint shards: 100%|██████████| 2/2 [00:01<00:00, 1.10it/s]\n",
"Trainer will use only 1 of 2 GPUs because it is running inside an interactive / notebook environment. You may try to set `Trainer(devices=2)` but please note that multi-GPU inside interactive / notebook environments is considered experimental and unstable. Your mileage may vary.\n",
"GPU available: True (cuda), used: True\n",
"TPU available: False, using: 0 TPU cores\n",
"IPU available: False, using: 0 IPUs\n",
"HPU available: False, using: 0 HPUs\n"
]
},
{
"ename": "TypeError",
"evalue": "Linear4bit.__init__() got an unexpected keyword argument 'dtype'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[18], line 29\u001b[0m\n\u001b[1;32m 18\u001b[0m trainer \u001b[38;5;241m=\u001b[39m pl\u001b[38;5;241m.\u001b[39mTrainer(\n\u001b[1;32m 19\u001b[0m accelerator\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgpu\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m 20\u001b[0m max_epochs\u001b[38;5;241m=\u001b[39mepochs,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 25\u001b[0m plugins\u001b[38;5;241m=\u001b[39mprecision\n\u001b[1;32m 26\u001b[0m )\n\u001b[1;32m 28\u001b[0m \u001b[38;5;66;03m# train\u001b[39;00m\n\u001b[0;32m---> 29\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpl_model\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdl_train\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdl_val\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 31\u001b[0m model \u001b[38;5;241m=\u001b[39m pl_model\u001b[38;5;241m.\u001b[39mmodel\n\u001b[1;32m 33\u001b[0m df_histe, df_hist \u001b[38;5;241m=\u001b[39m read_metrics_csv(trainer\u001b[38;5;241m.\u001b[39mlogger\u001b[38;5;241m.\u001b[39mexperiment\u001b[38;5;241m.\u001b[39mmetrics_file_path)\u001b[38;5;241m.\u001b[39mbfill()\u001b[38;5;241m.\u001b[39mffill()\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py:544\u001b[0m, in \u001b[0;36mTrainer.fit\u001b[0;34m(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)\u001b[0m\n\u001b[1;32m 542\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mstatus \u001b[38;5;241m=\u001b[39m TrainerStatus\u001b[38;5;241m.\u001b[39mRUNNING\n\u001b[1;32m 543\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m--> 544\u001b[0m \u001b[43mcall\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_and_handle_interrupt\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 545\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fit_impl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_dataloaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mval_dataloaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdatamodule\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mckpt_path\u001b[49m\n\u001b[1;32m 546\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/lightning/pytorch/trainer/call.py:44\u001b[0m, in \u001b[0;36m_call_and_handle_interrupt\u001b[0;34m(trainer, trainer_fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m trainer\u001b[38;5;241m.\u001b[39mstrategy\u001b[38;5;241m.\u001b[39mlauncher \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 43\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m trainer\u001b[38;5;241m.\u001b[39mstrategy\u001b[38;5;241m.\u001b[39mlauncher\u001b[38;5;241m.\u001b[39mlaunch(trainer_fn, \u001b[38;5;241m*\u001b[39margs, trainer\u001b[38;5;241m=\u001b[39mtrainer, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m---> 44\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtrainer_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 46\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m _TunerExitException:\n\u001b[1;32m 47\u001b[0m _call_teardown_hook(trainer)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py:580\u001b[0m, in \u001b[0;36mTrainer._fit_impl\u001b[0;34m(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)\u001b[0m\n\u001b[1;32m 573\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mfn \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 574\u001b[0m ckpt_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_checkpoint_connector\u001b[38;5;241m.\u001b[39m_select_ckpt_path(\n\u001b[1;32m 575\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mfn,\n\u001b[1;32m 576\u001b[0m ckpt_path,\n\u001b[1;32m 577\u001b[0m model_provided\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 578\u001b[0m model_connected\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlightning_module \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 579\u001b[0m )\n\u001b[0;32m--> 580\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_run\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mckpt_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mckpt_path\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 582\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mstopped\n\u001b[1;32m 583\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py:958\u001b[0m, in \u001b[0;36mTrainer._run\u001b[0;34m(self, model, ckpt_path)\u001b[0m\n\u001b[1;32m 955\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_checkpoint_connector\u001b[38;5;241m.\u001b[39m_restore_modules_and_callbacks(ckpt_path)\n\u001b[1;32m 957\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m: configuring model\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 958\u001b[0m \u001b[43mcall\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_configure_model\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 960\u001b[0m \u001b[38;5;66;03m# reset logger connector\u001b[39;00m\n\u001b[1;32m 961\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_logger_connector\u001b[38;5;241m.\u001b[39mreset_results()\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/lightning/pytorch/trainer/call.py:109\u001b[0m, in \u001b[0;36m_call_configure_model\u001b[0;34m(trainer)\u001b[0m\n\u001b[1;32m 107\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_overridden(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconfigure_model\u001b[39m\u001b[38;5;124m\"\u001b[39m, trainer\u001b[38;5;241m.\u001b[39mlightning_module):\n\u001b[1;32m 108\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m trainer\u001b[38;5;241m.\u001b[39mstrategy\u001b[38;5;241m.\u001b[39mtensor_init_context(), trainer\u001b[38;5;241m.\u001b[39mstrategy\u001b[38;5;241m.\u001b[39mmodel_sharded_context(), trainer\u001b[38;5;241m.\u001b[39mprecision_plugin\u001b[38;5;241m.\u001b[39mmodule_init_context(): \u001b[38;5;66;03m# noqa: E501\u001b[39;00m\n\u001b[0;32m--> 109\u001b[0m \u001b[43m_call_lightning_module_hook\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrainer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mconfigure_model\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/lightning/pytorch/trainer/call.py:157\u001b[0m, in \u001b[0;36m_call_lightning_module_hook\u001b[0;34m(trainer, hook_name, pl_module, *args, **kwargs)\u001b[0m\n\u001b[1;32m 154\u001b[0m pl_module\u001b[38;5;241m.\u001b[39m_current_fx_name \u001b[38;5;241m=\u001b[39m hook_name\n\u001b[1;32m 156\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m trainer\u001b[38;5;241m.\u001b[39mprofiler\u001b[38;5;241m.\u001b[39mprofile(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m[LightningModule]\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpl_module\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mhook_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m--> 157\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 159\u001b[0m \u001b[38;5;66;03m# restore current_fx when nested context\u001b[39;00m\n\u001b[1;32m 160\u001b[0m pl_module\u001b[38;5;241m.\u001b[39m_current_fx_name \u001b[38;5;241m=\u001b[39m prev_fx_name\n",
"Cell \u001b[0;32mIn[17], line 18\u001b[0m, in \u001b[0;36mPL_MODEL.configure_model\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mconfigure_model\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 10\u001b[0m \u001b[38;5;66;03m# instantiate your model in this hook\u001b[39;00m\n\u001b[1;32m 11\u001b[0m peft_config \u001b[38;5;241m=\u001b[39m LoraConfig(\n\u001b[1;32m 12\u001b[0m \u001b[38;5;66;03m# task_type=TaskType.TOKEN_CLS, \u001b[39;00m\n\u001b[1;32m 13\u001b[0m target_modules\u001b[38;5;241m=\u001b[39m[ \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfc2\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mWqkv\u001b[39m\u001b[38;5;124m\"\u001b[39m,],\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;66;03m# bias=\"all\"\u001b[39;00m\n\u001b[1;32m 17\u001b[0m )\n\u001b[0;32m---> 18\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel \u001b[38;5;241m=\u001b[39m \u001b[43mload_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 19\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel \u001b[38;5;241m=\u001b[39m get_peft_model(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel, peft_config)\n\u001b[1;32m 20\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39muse_cache \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n",
"Cell \u001b[0;32mIn[5], line 5\u001b[0m, in \u001b[0;36mload_model\u001b[0;34m()\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload_model\u001b[39m():\n\u001b[0;32m----> 5\u001b[0m model \u001b[38;5;241m=\u001b[39m \u001b[43mAutoModelForCausalLM\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 6\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# torch_dtype=torch.float16,\u001b[39;49;00m\n\u001b[1;32m 8\u001b[0m \u001b[43m \u001b[49m\u001b[43mtrust_remote_code\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 9\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m model\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/models/auto/auto_factory.py:560\u001b[0m, in \u001b[0;36m_BaseAutoModelClass.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m 558\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 559\u001b[0m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39mregister(config\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m, model_class, exist_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m--> 560\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmodel_class\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 561\u001b[0m \u001b[43m \u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mhub_kwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\n\u001b[1;32m 562\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 563\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mtype\u001b[39m(config) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_model_mapping\u001b[38;5;241m.\u001b[39mkeys():\n\u001b[1;32m 564\u001b[0m model_class \u001b[38;5;241m=\u001b[39m _get_model_class(config, \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_model_mapping)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/modeling_utils.py:3085\u001b[0m, in \u001b[0;36mPreTrainedModel.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m 3082\u001b[0m config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_check_and_enable_flash_attn_2(config, torch_dtype\u001b[38;5;241m=\u001b[39mtorch_dtype, device_map\u001b[38;5;241m=\u001b[39mdevice_map)\n\u001b[1;32m 3084\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m ContextManagers(init_contexts):\n\u001b[0;32m-> 3085\u001b[0m model \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3087\u001b[0m \u001b[38;5;66;03m# Check first if we are `from_pt`\u001b[39;00m\n\u001b[1;32m 3088\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m use_keep_in_fp32_modules:\n",
"File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:933\u001b[0m, in \u001b[0;36mPhiForCausalLM.__init__\u001b[0;34m(self, config)\u001b[0m\n\u001b[1;32m 930\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, config: PhiConfig) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 931\u001b[0m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(config)\n\u001b[0;32m--> 933\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtransformer \u001b[38;5;241m=\u001b[39m \u001b[43mPhiModel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 934\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlm_head \u001b[38;5;241m=\u001b[39m CausalLMHead(config)\n\u001b[1;32m 935\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mloss \u001b[38;5;241m=\u001b[39m CausalLMLoss()\n",
"File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:896\u001b[0m, in \u001b[0;36mPhiModel.__init__\u001b[0;34m(self, config)\u001b[0m\n\u001b[1;32m 893\u001b[0m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(config)\n\u001b[1;32m 895\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membd \u001b[38;5;241m=\u001b[39m Embedding(config)\n\u001b[0;32m--> 896\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mh \u001b[38;5;241m=\u001b[39m nn\u001b[38;5;241m.\u001b[39mModuleList(\u001b[43m[\u001b[49m\u001b[43mParallelBlock\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mblock_idx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mi\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mi\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mrange\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mn_layer\u001b[49m\u001b[43m)\u001b[49m\u001b[43m]\u001b[49m)\n\u001b[1;32m 897\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgradient_checkpointing \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 898\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpost_init()\n",
"File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:896\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 893\u001b[0m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(config)\n\u001b[1;32m 895\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membd \u001b[38;5;241m=\u001b[39m Embedding(config)\n\u001b[0;32m--> 896\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mh \u001b[38;5;241m=\u001b[39m nn\u001b[38;5;241m.\u001b[39mModuleList([\u001b[43mParallelBlock\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mblock_idx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mi\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(config\u001b[38;5;241m.\u001b[39mn_layer)])\n\u001b[1;32m 897\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgradient_checkpointing \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 898\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpost_init()\n",
"File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:757\u001b[0m, in \u001b[0;36mParallelBlock.__init__\u001b[0;34m(self, config, block_idx)\u001b[0m\n\u001b[1;32m 754\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mresid_dropout \u001b[38;5;241m=\u001b[39m nn\u001b[38;5;241m.\u001b[39mDropout(config\u001b[38;5;241m.\u001b[39mresid_pdrop)\n\u001b[1;32m 755\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mblock_idx \u001b[38;5;241m=\u001b[39m block_idx\n\u001b[0;32m--> 757\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmixer \u001b[38;5;241m=\u001b[39m \u001b[43mMHA\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlayer_idx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mblock_idx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 758\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmlp \u001b[38;5;241m=\u001b[39m MLP(config)\n",
"File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:562\u001b[0m, in \u001b[0;36mMHA.__init__\u001b[0;34m(self, config, dtype, device, rotary_dim, rotary_base, rotary_scale_base, n_head, n_head_kv, head_dim, bias, causal, softmax_scale, layer_idx, return_residual, checkpointing)\u001b[0m\n\u001b[1;32m 559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m linear_cls \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 560\u001b[0m linear_cls \u001b[38;5;241m=\u001b[39m nn\u001b[38;5;241m.\u001b[39mLinear\n\u001b[0;32m--> 562\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mWqkv \u001b[38;5;241m=\u001b[39m \u001b[43mlinear_cls\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mop_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbias\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 563\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mout_proj \u001b[38;5;241m=\u001b[39m linear_cls(hidden_size, hidden_size, bias\u001b[38;5;241m=\u001b[39mbias, device\u001b[38;5;241m=\u001b[39mdevice, dtype\u001b[38;5;241m=\u001b[39mdtype)\n\u001b[1;32m 565\u001b[0m \u001b[38;5;66;03m# Attention\u001b[39;00m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/lightning/fabric/plugins/precision/bitsandbytes.py:253\u001b[0m, in \u001b[0;36m_import_bitsandbytes.<locals>._NF4DQLinear.__init__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs: Any, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 253\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mquant_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mnf4\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcompress_statistics\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/lightning/fabric/plugins/precision/bitsandbytes.py:213\u001b[0m, in \u001b[0;36m_import_bitsandbytes.<locals>._Linear4bit.__init__\u001b[0;34m(self, device, *args, **kwargs)\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs: Any, device: Optional[_DEVICE] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 213\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 214\u001b[0m \u001b[38;5;66;03m# if the device is CUDA or we are under a CUDA context manager, quantize the weight here, so we don't end up\u001b[39;00m\n\u001b[1;32m 215\u001b[0m \u001b[38;5;66;03m# filling the device memory with float32 weights which could lead to OOM\u001b[39;00m\n\u001b[1;32m 216\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mtensor(\u001b[38;5;241m0\u001b[39m, device\u001b[38;5;241m=\u001b[39mdevice)\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mtype \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcuda\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n",
"\u001b[0;31mTypeError\u001b[0m: Linear4bit.__init__() got an unexpected keyword argument 'dtype'"
]
}
],
"source": [
"\n",
" \n",
"\n",
"\n",
"\n",
"device = 'cuda'\n",
"lr = 4e-3\n",
"epochs = 3\n",
"accum_steps = 16\n",
"batch_size = 2\n",
"\n",
"Xs, Ys, attention_masks = str2xya(first_half, tokenizer)\n",
"dl_train = DataLoader(TensorDataset(Xs, Ys, attention_masks), batch_size=batch_size, shuffle=True)\n",
"Xs, Ys, attention_masks = str2xya(second_half, tokenizer)\n",
"dl_val = DataLoader(TensorDataset(Xs, Ys, attention_masks), batch_size=batch_size, shuffle=False)\n",
"\n",
"epoch_steps = len(dl_train)\n",
"\n",
"pl_model = PL_MODEL(num_iterations=epoch_steps*epochs, lr=lr, weight_decay=0)\n",
"from lightning.pytorch.plugins import BitsandbytesPrecision\n",
"precision = BitsandbytesPrecision(mode=\"nf4-dq\")\n",
"# precision = BitsandbytesPrecision(mode=\"int8-training\", dtype=torch.float16, ignore_modules={\"lm_head\"})\n",
"trainer = pl.Trainer(\n",
" accelerator='gpu',\n",
" max_epochs=epochs,\n",
" # precision='',\n",
" # precision=\"bf16-mixed\",\n",
" log_every_n_steps=1,\n",
" accumulate_grad_batches=accum_steps,\n",
" plugins=precision\n",
" )\n",
"\n",
"# train\n",
"trainer.fit(pl_model, dl_train, dl_val)\n",
"\n",
"model = pl_model.model\n",
"\n",
"df_histe, df_hist = read_metrics_csv(trainer.logger.experiment.metrics_file_path).bfill().ffill()\n",
"display(df_hist)\n",
"plot_hist(df_hist)\n",
"\n",
"eval(model, tokenizer, second_half)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"1/0"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Old"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from torch import optim\n",
"\n",
"\n",
"def lora_eval(model, tokenizer, sample):\n",
" # reset/set adapter\n",
" # peft_config = IA3Config(\n",
" # target_modules=[ \"fc2\", \"Wqkv\",], \n",
" # feedforward_modules=[\"fc2\"],\n",
" # inference_mode=False,\n",
" # )\n",
" peft_config = LoraConfig(\n",
" # task_type=TaskType.TOKEN_CLS, \n",
" target_modules=[ \"fc2\", \"Wqkv\",],\n",
" inference_mode=False, r=16, lora_alpha=16, lora_dropout=0.1, bias=\"all\"\n",
" )\n",
" model = get_peft_model(model, peft_config)\n",
" model.config.use_cache = False\n",
"\n",
" # train adapter\n",
" s = sample['text']\n",
" first_half = s[:len(s)//2]\n",
" second_half = s[len(s)//2:]\n",
" input_ids = tokenizer(first_half, return_tensors=\"pt\")[\"input_ids\"][0].to('cuda')\n",
"\n",
" device = 'cuda'\n",
" lr = 1.0e-2\n",
" epochs = 3\n",
" accum_steps = 64\n",
" epoch_steps = (len(input_ids)-1)//accum_steps+1\n",
"\n",
" total_steps = epochs * epoch_steps\n",
" optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n",
" scheduler = optim.lr_scheduler.OneCycleLR(\n",
" optimizer, lr, total_steps=total_steps\n",
" )\n",
" model.train()\n",
" model = model.to(device)\n",
" for epoch in range(epochs):\n",
" # TODO: batch\n",
" \n",
" accum = 0\n",
" for i in range(1, len(input_ids)):\n",
" X = input_ids[:i][None, ]\n",
" targets = input_ids[i:i+1][None, ]\n",
" optimizer.zero_grad()\n",
" out = model(input_ids=X, \n",
" )\n",
" logits = out['logits'][:, -1]\n",
" loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))\n",
" loss.backward()\n",
" if accum > accum_steps:\n",
" optimizer.step()\n",
" scheduler.step()\n",
" optimizer.zero_grad()\n",
" accum = 0\n",
" else:\n",
" accum += 1\n",
" if accum > 0:\n",
" optimizer.step()\n",
" scheduler.step()\n",
" optimizer.zero_grad()\n",
"\n",
" return eval(model, tokenizer, second_half)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"data = []\n",
"for sample in tqdm(samples):\n",
" r = lora_eval(model, tokenizer, sample)\n",
" print(sample['name'], r)\n",
" r.update(sample)\n",
" data.append(r)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print('perplexity (on 2nd half) before and after training adapter on first half of text')\n",
"df = pd.DataFrame(data).set_index('name')\n",
"\n",
"df['learning'] = (df['before']-df['after'])/df['before']\n",
"df.sort_values('learning').drop(columns=['text', 'url'])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Result"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.0rc1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
nbs/02_use_lora_transformertrain.ipynb
-516
View File
@@ -1,516 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"https://github.com/huggingface/peft/blob/main/examples/fp4_finetuning/finetune_fp4_opt_bnb_peft.py"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"from torch import optim\n",
"import lightning as pl\n",
"from matplotlib import pyplot as plt"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"import torch\n",
"import torch.nn as nn\n",
"import transformers\n",
"from datasets import load_dataset\n",
"from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, AutoConfig\n",
"import numpy as np\n",
"from tqdm.auto import tqdm\n",
"import pandas as pd\n",
"import warnings\n",
"from peft import LoraConfig, get_peft_model, IA3Config"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"plt.style.use('ggplot')\n",
"torch.set_float32_matmul_precision('medium')\n",
"warnings.filterwarnings(\"ignore\", \".*does not have many workers.*\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
"\n",
"model_name = \"microsoft/phi-2\"\n",
"\n",
"# model = AutoModelForCausalLM.from_pretrained(\n",
"# model_name,\n",
"# # max_memory=max_memory,\n",
"# quantization_config=BitsAndBytesConfig(\n",
"# load_in_4bit=True,\n",
"# llm_int8_threshold=6.0,\n",
"# llm_int8_has_fp16_weight=False,\n",
"# bnb_4bit_compute_dtype=torch.float16,\n",
"# bnb_4bit_use_double_quant=True,\n",
"# bnb_4bit_quant_type=\"nf4\",\n",
"# ),\n",
"# torch_dtype=torch.float16,\n",
"# trust_remote_code=True,\n",
"# )\n",
"\n",
"\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"model_name = \"TheBloke/phi-2-GPTQ\"\n",
"# model_name = \"microsoft/phi-2\"\n",
"\n",
"def load_model():\n",
"\n",
" # model = AutoModelForCausalLM.from_pretrained(\n",
" # model_name,\n",
" # # quantization_config=BitsAndBytesConfig(\n",
" # # load_in_4bit=True,\n",
" # # llm_int8_threshold=6.0,\n",
" # # llm_int8_has_fp16_weight=False,\n",
" # # bnb_4bit_compute_dtype=torch.float16,\n",
" # # bnb_4bit_use_double_quant=True,\n",
" # # bnb_4bit_quant_type=\"nf4\",\n",
" # # ),\n",
" # torch_dtype=torch.float16,\n",
" # trust_remote_code=True,\n",
" # )\n",
"\n",
"\n",
" config = AutoConfig.from_pretrained(model_name, trust_remote_code=True,)\n",
" config.quantization_config['use_exllama'] = False\n",
" config.quantization_config['disable_exllama'] = True\n",
" model = AutoModelForCausalLM.from_pretrained(\n",
" model_name,\n",
" torch_dtype=torch.bfloat16,\n",
" trust_remote_code=True,\n",
" config=config,\n",
" )\n",
" return model\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"CUDA extension not installed.\n",
"CUDA extension not installed.\n",
"Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
]
}
],
"source": [
"model = load_model()\n",
"tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True,)\n",
"tokenizer.pad_token = tokenizer.eos_token"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"peft_config = LoraConfig(\n",
" # task_type=TaskType.TOKEN_CLS, \n",
" target_modules=[ \"fc2\", \"Wqkv\",],\n",
" inference_mode=False, r=16, lora_alpha=16, lora_dropout=0.1, bias=\"all\"\n",
")\n",
"model = get_peft_model(model, peft_config)\n",
"model.config.use_cache = False"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"MAX_LEN = 2000\n",
"samples = json.load(open(\"../samples.json\"))\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Helpers"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"# modified from https://github.dev/huggingface/evaluate/blob/8dfe05784099fb9af55b8e77793205a3b7c86465/measurements/perplexity/perplexity.py#L154\n",
"\n",
"# from evaluate.measurements.perplexity import Perplexity\n",
"import evaluate\n",
"from evaluate import logging\n",
"from torch.nn import CrossEntropyLoss\n",
"\n",
"# @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)\n",
"def perplexity_compute(\n",
" data, model, tokenizer, batch_size: int = 16, add_start_token: bool = True, device=None, max_length=None\n",
"):\n",
"\n",
" if device is not None:\n",
" assert device in [\"gpu\", \"cpu\", \"cuda\"], \"device should be either gpu or cpu.\"\n",
" if device == \"gpu\":\n",
" device = \"cuda\"\n",
" else:\n",
" device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
"\n",
" # model = AutoModelForCausalLM.from_pretrained(model_id)\n",
" model = model.to(device)\n",
"\n",
" # tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
"\n",
" # if batch_size > 1 (which generally leads to padding being required), and\n",
" # if there is not an already assigned pad_token, assign an existing\n",
" # special token to also be the padding token\n",
" if tokenizer.pad_token is None and batch_size > 1:\n",
" existing_special_tokens = list(tokenizer.special_tokens_map_extended.values())\n",
" # check that the model already has at least one special token defined\n",
" assert (\n",
" len(existing_special_tokens) > 0\n",
" ), \"If batch_size > 1, model must have at least one special token to use for padding. Please use a different model or set batch_size=1.\"\n",
" # assign one of the special tokens to also be the pad token\n",
" tokenizer.add_special_tokens({\"pad_token\": existing_special_tokens[0]})\n",
"\n",
" if add_start_token and max_length:\n",
" # leave room for <BOS> token to be added:\n",
" assert (\n",
" tokenizer.bos_token is not None\n",
" ), \"Input model must already have a BOS token if using add_start_token=True. Please use a different model, or set add_start_token=False\"\n",
" max_tokenized_len = max_length - 1\n",
" else:\n",
" max_tokenized_len = max_length\n",
"\n",
" encodings = tokenizer(\n",
" data,\n",
" add_special_tokens=False,\n",
" padding=True,\n",
" truncation=True if max_tokenized_len else False,\n",
" max_length=max_tokenized_len,\n",
" return_tensors=\"pt\",\n",
" return_attention_mask=True,\n",
" ).to(device)\n",
"\n",
" encoded_texts = encodings[\"input_ids\"]\n",
" attn_masks = encodings[\"attention_mask\"]\n",
"\n",
" # check that each input is long enough:\n",
" if add_start_token:\n",
" assert torch.all(torch.ge(attn_masks.sum(1), 1)), \"Each input text must be at least one token long.\"\n",
" else:\n",
" assert torch.all(\n",
" torch.ge(attn_masks.sum(1), 2)\n",
" ), \"When add_start_token=False, each input text must be at least two tokens long. Run with add_start_token=True if inputting strings of only one token, and remove all empty input strings.\"\n",
"\n",
" ppls = []\n",
" loss_fct = CrossEntropyLoss(reduction=\"none\")\n",
"\n",
" for start_index in logging.tqdm(range(0, len(encoded_texts), batch_size)):\n",
" end_index = min(start_index + batch_size, len(encoded_texts))\n",
" encoded_batch = encoded_texts[start_index:end_index]\n",
" attn_mask = attn_masks[start_index:end_index]\n",
"\n",
" if add_start_token:\n",
" bos_tokens_tensor = torch.tensor([[tokenizer.bos_token_id]] * encoded_batch.size(dim=0)).to(device)\n",
" encoded_batch = torch.cat([bos_tokens_tensor, encoded_batch], dim=1)\n",
" attn_mask = torch.cat(\n",
" [torch.ones(bos_tokens_tensor.size(), dtype=torch.int64).to(device), attn_mask], dim=1\n",
" )\n",
"\n",
" labels = encoded_batch\n",
"\n",
" with torch.no_grad():\n",
" out_logits = model(encoded_batch, attention_mask=attn_mask).logits\n",
"\n",
" shift_logits = out_logits[..., :-1, :].contiguous()\n",
" shift_labels = labels[..., 1:].contiguous()\n",
" shift_attention_mask_batch = attn_mask[..., 1:].contiguous()\n",
"\n",
" perplexity_batch = torch.exp(\n",
" (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch).sum(1)\n",
" / shift_attention_mask_batch.sum(1)\n",
" )\n",
"\n",
" ppls += perplexity_batch.tolist()\n",
"\n",
" return {\"perplexities\": ppls, \"mean_perplexity\": torch.tensor(ppls).mean()}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Training"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"from torch.nn import functional as F\n",
"from torch.utils.data import DataLoader, TensorDataset"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Lightning helpers"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"sample = samples[0]\n",
"s = sample['text']\n",
"first_half = s[:len(s)//2]\n",
"second_half = s[len(s)//2:]\n",
"\n",
"\n",
"\n",
"def str2xya(s, tokenizer):\n",
" max_len = min(MAX_LEN, len(s))\n",
" input_ids = tokenizer(s, return_tensors=\"pt\")[\"input_ids\"][0]\n",
"\n",
" pad = tokenizer.bos_token_id\n",
" data = []\n",
" for i in range(1, len(input_ids)):\n",
" x = input_ids[:i][-max_len:]\n",
" padding = max_len - len(x)\n",
" x = torch.tensor([pad]*padding + x.tolist())\n",
"\n",
" label_ids = input_ids[i:i+1]\n",
" attention_mask = (x==pad)*1\n",
" data.append(dict(input_ids=x, label_ids=label_ids, attention_mask=attention_mask))\n",
" \n",
" return data\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"def eval(model, tokenizer, second_half):\n",
" model.eval();\n",
" with torch.no_grad():\n",
" with model.disable_adapter():\n",
" results = perplexity_compute(data=second_half, model=model, tokenizer=tokenizer, device='cuda')\n",
" results2 = perplexity_compute(data=second_half, model=model, tokenizer=tokenizer, device='cuda')\n",
" return dict(before=results['mean_perplexity'].item(), after=results2['mean_perplexity'].item())\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Train"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"from datasets import Dataset\n",
"\n",
"# data_train[0]"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"device = 'cuda'\n",
"lr = 4e-3\n",
"epochs = 3\n",
"accum_steps = 16\n",
"batch_size = 1\n",
"\n",
"data_train = str2xya(first_half, tokenizer)\n",
"ds_train = Dataset.from_list(data_train).with_format(\"torch\")\n",
"dl_train = DataLoader(ds_train, batch_size=batch_size, shuffle=True)\n",
"data_val = str2xya(second_half, tokenizer)\n",
"ds_val = Dataset.from_list(data_val).with_format(\"torch\")\n",
"dl_val = DataLoader(ds_val, batch_size=batch_size, shuffle=False)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"# class CustomTrainer(transformers.Trainer):\n",
"# def compute_loss(self, model, inputs, return_outputs=False):\n",
"# labels = inputs.pop(\"labels\")\n",
"# # forward pass\n",
"# outputs = model(**inputs)\n",
"# logits = outputs.get(\"logits\")\n",
"# # compute custom loss (suppose one has 3 labels with different weights)\n",
"# loss_fct = nn.CrossEntropyLoss(weight=torch.tensor([1.0, 2.0, 3.0], device=model.device))\n",
"# loss = loss_fct(logits.view(-1, self.model.config.num_labels), labels.view(-1))\n",
"# return (loss, outputs) if return_outputs else loss"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/parallel/data_parallel.py:33: UserWarning: \n",
" There is an imbalance between your GPUs. You may want to exclude GPU 1 which\n",
" has less than 75% of the memory or cores of GPU 0. You can do so by setting\n",
" the device_ids argument to DataParallel, or by setting the CUDA_VISIBLE_DEVICES\n",
" environment variable.\n",
" warnings.warn(imbalance_warn.format(device_ids[min_pos], device_ids[max_pos]))\n",
" 0%| | 0/20 [00:00<?, ?it/s]You're using a CodeGenTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
]
},
{
"ename": "OutOfMemoryError",
"evalue": "Caught OutOfMemoryError in replica 1 on device 1.\nOriginal Traceback (most recent call last):\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/parallel/parallel_apply.py\", line 85, in _worker\n output = module(*input, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/peft/peft_model.py\", line 536, in forward\n return self.get_base_model()(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/wassname/.cache/huggingface/modules/transformers_modules/TheBloke/phi-2-GPTQ/8c61fa56159b3e69e768afcbebaf14bdb11532f2/modeling_phi.py\", line 953, in forward\n hidden_states = self.transformer(input_ids, past_key_values=past_key_values, attention_mask=attention_mask)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/wassname/.cache/huggingface/modules/transformers_modules/TheBloke/phi-2-GPTQ/8c61fa56159b3e69e768afcbebaf14bdb11532f2/modeling_phi.py\", line 915, in forward\n hidden_states = layer(\n ^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/wassname/.cache/huggingface/modules/transformers_modules/TheBloke/phi-2-GPTQ/8c61fa56159b3e69e768afcbebaf14bdb11532f2/modeling_phi.py\", line 770, in forward\n attn_outputs = self.mixer(\n ^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/wassname/.cache/huggingface/modules/transformers_modules/TheBloke/phi-2-GPTQ/8c61fa56159b3e69e768afcbebaf14bdb11532f2/modeling_phi.py\", line 722, in forward\n attn_output = self._forward_self_attn(x, attention_mask)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/wassname/.cache/huggingface/modules/transformers_modules/TheBloke/phi-2-GPTQ/8c61fa56159b3e69e768afcbebaf14bdb11532f2/modeling_phi.py\", line 621, in _forward_self_attn\n return self.inner_attn(qkv, key_padding_mask=key_padding_mask)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n return forward_calLine truncated
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mOutOfMemoryError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[16], line 21\u001b[0m\n\u001b[1;32m 19\u001b[0m trainer\u001b[38;5;241m.\u001b[39m_signature_columns \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124m'\u001b[39m\u001b[38;5;124minput_ids\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mattention_mask\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlabel_ids\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m 20\u001b[0m model\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39muse_cache \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m \u001b[38;5;66;03m# silence the warnings. Please re-enable for inference!\u001b[39;00m\n\u001b[0;32m---> 21\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/trainer.py:1591\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m 1589\u001b[0m hf_hub_utils\u001b[38;5;241m.\u001b[39menable_progress_bars()\n\u001b[1;32m 1590\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1591\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43minner_training_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1592\u001b[0m \u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1593\u001b[0m \u001b[43m \u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1594\u001b[0m \u001b[43m \u001b[49m\u001b[43mtrial\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1595\u001b[0m \u001b[43m \u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1596\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/trainer.py:1892\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m 1889\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcallback_handler\u001b[38;5;241m.\u001b[39mon_step_begin(args, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol)\n\u001b[1;32m 1891\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maccelerator\u001b[38;5;241m.\u001b[39maccumulate(model):\n\u001b[0;32m-> 1892\u001b[0m tr_loss_step \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtraining_step\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1894\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m 1895\u001b[0m args\u001b[38;5;241m.\u001b[39mlogging_nan_inf_filter\n\u001b[1;32m 1896\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_torch_tpu_available()\n\u001b[1;32m 1897\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m (torch\u001b[38;5;241m.\u001b[39misnan(tr_loss_step) \u001b[38;5;129;01mor\u001b[39;00m torch\u001b[38;5;241m.\u001b[39misinf(tr_loss_step))\n\u001b[1;32m 1898\u001b[0m ):\n\u001b[1;32m 1899\u001b[0m \u001b[38;5;66;03m# if loss is nan or inf simply add the average of previous logged losses\u001b[39;00m\n\u001b[1;32m 1900\u001b[0m tr_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m tr_loss \u001b[38;5;241m/\u001b[39m (\u001b[38;5;241m1\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mglobal_step \u001b[38;5;241m-\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_globalstep_last_logged)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/trainer.py:2776\u001b[0m, in \u001b[0;36mTrainer.training_step\u001b[0;34m(self, model, inputs)\u001b[0m\n\u001b[1;32m 2773\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m loss_mb\u001b[38;5;241m.\u001b[39mreduce_mean()\u001b[38;5;241m.\u001b[39mdetach()\u001b[38;5;241m.\u001b[39mto(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mdevice)\n\u001b[1;32m 2775\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcompute_loss_context_manager():\n\u001b[0;32m-> 2776\u001b[0m loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcompute_loss\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2778\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mn_gpu \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[1;32m 2779\u001b[0m loss \u001b[38;5;241m=\u001b[39m loss\u001b[38;5;241m.\u001b[39mmean() \u001b[38;5;66;03m# mean() to average on multi-gpu parallel training\u001b[39;00m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/trainer.py:2801\u001b[0m, in \u001b[0;36mTrainer.compute_loss\u001b[0;34m(self, model, inputs, return_outputs)\u001b[0m\n\u001b[1;32m 2799\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 2800\u001b[0m labels \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m-> 2801\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2802\u001b[0m \u001b[38;5;66;03m# Save past state if it exists\u001b[39;00m\n\u001b[1;32m 2803\u001b[0m \u001b[38;5;66;03m# TODO: this needs to be fixed and made cleaner later.\u001b[39;00m\n\u001b[1;32m 2804\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mpast_index \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/parallel/data_parallel.py:185\u001b[0m, in \u001b[0;36mDataParallel.forward\u001b[0;34m(self, *inputs, **kwargs)\u001b[0m\n\u001b[1;32m 183\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodule(\u001b[38;5;241m*\u001b[39minputs[\u001b[38;5;241m0\u001b[39m], \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mmodule_kwargs[\u001b[38;5;241m0\u001b[39m])\n\u001b[1;32m 184\u001b[0m replicas \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreplicate(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodule, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice_ids[:\u001b[38;5;28mlen\u001b[39m(inputs)])\n\u001b[0;32m--> 185\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparallel_apply\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreplicas\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodule_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 186\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgather(outputs, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutput_device)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/parallel/data_parallel.py:200\u001b[0m, in \u001b[0;36mDataParallel.parallel_apply\u001b[0;34m(self, replicas, inputs, kwargs)\u001b[0m\n\u001b[1;32m 199\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mparallel_apply\u001b[39m(\u001b[38;5;28mself\u001b[39m, replicas: Sequence[T], inputs: Sequence[Any], kwargs: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m List[Any]:\n\u001b[0;32m--> 200\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mparallel_apply\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreplicas\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdevice_ids\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mreplicas\u001b[49m\u001b[43m)\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/parallel/parallel_apply.py:110\u001b[0m, in \u001b[0;36mparallel_apply\u001b[0;34m(modules, inputs, kwargs_tup, devices)\u001b[0m\n\u001b[1;32m 108\u001b[0m output \u001b[38;5;241m=\u001b[39m results[i]\n\u001b[1;32m 109\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(output, ExceptionWrapper):\n\u001b[0;32m--> 110\u001b[0m \u001b[43moutput\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreraise\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 111\u001b[0m outputs\u001b[38;5;241m.\u001b[39mappend(output)\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m outputs\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/_utils.py:694\u001b[0m, in \u001b[0;36mExceptionWrapper.reraise\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 690\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[1;32m 691\u001b[0m \u001b[38;5;66;03m# If the exception takes multiple arguments, don't try to\u001b[39;00m\n\u001b[1;32m 692\u001b[0m \u001b[38;5;66;03m# instantiate since we don't know how to\u001b[39;00m\n\u001b[1;32m 693\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(msg) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 694\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exception\n",
"\u001b[0;31mOutOfMemoryError\u001b[0m: Caught OutOfMemoryError in replica 1 on device 1.\nOriginal Traceback (most recent call last):\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/parallel/parallel_apply.py\", line 85, in _worker\n output = module(*input, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/peft/peft_model.py\", line 536, in forward\n return self.get_base_model()(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/wassname/.cache/huggingface/modules/transformers_modules/TheBloke/phi-2-GPTQ/8c61fa56159b3e69e768afcbebaf14bdb11532f2/modeling_phi.py\", line 953, in forward\n hidden_states = self.transformer(input_ids, past_key_values=past_key_values, attention_mask=attention_mask)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/wassname/.cache/huggingface/modules/transformers_modules/TheBloke/phi-2-GPTQ/8c61fa56159b3e69e768afcbebaf14bdb11532f2/modeling_phi.py\", line 915, in forward\n hidden_states = layer(\n ^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/wassname/.cache/huggingface/modules/transformers_modules/TheBloke/phi-2-GPTQ/8c61fa56159b3e69e768afcbebaf14bdb11532f2/modeling_phi.py\", line 770, in forward\n attn_outputs = self.mixer(\n ^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/wassname/.cache/huggingface/modules/transformers_modules/TheBloke/phi-2-GPTQ/8c61fa56159b3e69e768afcbebaf14bdb11532f2/modeling_phi.py\", line 722, in forward\n attn_output = self._forward_self_attn(x, attention_mask)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/wassname/.cache/huggingface/modules/transformers_modules/TheBloke/phi-2-GPTQ/8c61fa56159b3e69e768afcbebaf14bdb11532f2/modeling_phi.py\", line 621, in _forward_self_attn\n return self.inner_attn(qkv, key_padding_mask=key_padding_mask)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1527, in _calLine truncated
]
}
],
"source": [
"# tokenizer.pad_token = tokenizer.eos_token\n",
"os.environ['CUDA_VISIBLE_DEVICES']=\"1\"\n",
"trainer = transformers.Trainer(\n",
" model=model,\n",
" train_dataset=ds_train,\n",
" eval_dataset=ds_val,\n",
" args=transformers.TrainingArguments(\n",
" per_device_train_batch_size=batch_size,\n",
" gradient_accumulation_steps=4,\n",
" warmup_steps=10,\n",
" max_steps=20,\n",
" learning_rate=3e-4,\n",
" fp16=True,\n",
" logging_steps=1,\n",
" output_dir=\"outputs\",\n",
" ),\n",
" # data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),\n",
")\n",
"trainer._signature_columns = ['input_ids', 'attention_mask', 'label_ids']\n",
"model.config.use_cache = False # silence the warnings. Please re-enable for inference!\n",
"trainer.train()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.0rc1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
nbs/02_use_lora_transformertrain_WORK.ipynb
-1611
View File
File diff suppressed because it is too large. Load diff
nbs/03_use_ia3_hftrain .ipynb
-6106
View File
File diff suppressed because it is too large. Load diff
nbs/03_use_lora_hftrain.ipynb
-4094
View File
File diff suppressed because it is too large. Load diff