From dfaa00dccc1038e1ad8aa00d5868d80d1598fa4a Mon Sep 17 00:00:00 2001 From: Sotirios Anagnostidis Date: Thu, 5 Jan 2023 00:33:16 +0100 Subject: [PATCH 01/32] gptj 8bit --- .../supervised_finetuning/configs/config.yaml | 5 + .../supervised_finetuning/models/__init__.py | 8 + model/supervised_finetuning/models/gptj.py | 185 ++++++++++++++++++ model/supervised_finetuning/trainer.py | 17 +- model/supervised_finetuning/utils.py | 5 +- 5 files changed, 215 insertions(+), 5 deletions(-) create mode 100644 model/supervised_finetuning/models/__init__.py create mode 100644 model/supervised_finetuning/models/gptj.py diff --git a/model/supervised_finetuning/configs/config.yaml b/model/supervised_finetuning/configs/config.yaml index 29086395..ded4363e 100644 --- a/model/supervised_finetuning/configs/config.yaml +++ b/model/supervised_finetuning/configs/config.yaml @@ -21,6 +21,7 @@ defaults: loss_fn: CrossEntropyLoss eval_size: log_dir: "base" + quantization: galactica-125: learning_rate: 5e-5 @@ -46,3 +47,7 @@ gpt-jt: debug: eval_steps: 20 eval_size: 100 + gradient_accumulation_steps: 2 + per_device_train_batch_size: 1 + per_device_eval_batch_size: 1 + quantization: 8bit \ No newline at end of file diff --git a/model/supervised_finetuning/models/__init__.py b/model/supervised_finetuning/models/__init__.py new file mode 100644 index 00000000..673f4085 --- /dev/null +++ b/model/supervised_finetuning/models/__init__.py @@ -0,0 +1,8 @@ +from transformers import AutoModelForCausalLM +from .gptj import get_model as get_gptj_model + +def get_specific_model(model_name, cache_dir, quantization): + if "gpt-j" in model_name.lower(): + return get_gptj_model(model_name, cache_dir, quantization) + else: + return AutoModelForCausalLM.from_pretrained(conf.model_name, cache_dir=conf.cache_dir) \ No newline at end of file diff --git a/model/supervised_finetuning/models/gptj.py b/model/supervised_finetuning/models/gptj.py new file mode 100644 index 00000000..11234abd --- /dev/null +++ b/model/supervised_finetuning/models/gptj.py @@ -0,0 +1,185 @@ +# Taken from https://github.com/sleekmike/Finetune_GPT-J_6B_8-bit/blob/master/gpt-j-6b-8-bit.py + +import transformers +from transformers import AutoModelForCausalLM +import torch +import torch.nn.functional as F +from torch import nn +from torch.cuda.amp import custom_fwd, custom_bwd +from bitsandbytes.functional import quantize_blockwise, dequantize_blockwise + + +class FrozenBNBLinear(nn.Module): + def __init__(self, weight, absmax, code, bias=None): + assert isinstance(bias, nn.Parameter) or bias is None + super().__init__() + self.out_features, self.in_features = weight.shape + self.register_buffer("weight", weight.requires_grad_(False)) + self.register_buffer("absmax", absmax.requires_grad_(False)) + self.register_buffer("code", code.requires_grad_(False)) + self.adapter = None + self.bias = bias + + def forward(self, input): + output = DequantizeAndLinear.apply(input, self.weight, self.absmax, self.code, self.bias) + if self.adapter: + output += self.adapter(input) + return output + + @classmethod + def from_linear(cls, linear: nn.Linear) -> "FrozenBNBLinear": + weights_int8, state = quantize_blockise_lowmemory(linear.weight) + return cls(weights_int8, *state, linear.bias) + + def __repr__(self): + return f"{self.__class__.__name__}({self.in_features}, {self.out_features})" + + +class DequantizeAndLinear(torch.autograd.Function): + @staticmethod + @custom_fwd + def forward( + ctx, + input: torch.Tensor, + weights_quantized: torch.ByteTensor, + absmax: torch.FloatTensor, + code: torch.FloatTensor, + bias: torch.FloatTensor, + ): + weights_deq = dequantize_blockwise(weights_quantized, absmax=absmax, code=code) + ctx.save_for_backward(input, weights_quantized, absmax, code) + ctx._has_bias = bias is not None + return F.linear(input, weights_deq, bias) + + @staticmethod + @custom_bwd + def backward(ctx, grad_output: torch.Tensor): + assert not ctx.needs_input_grad[1] and not ctx.needs_input_grad[2] and not ctx.needs_input_grad[3] + input, weights_quantized, absmax, code = ctx.saved_tensors + # grad_output: [*batch, out_features] + weights_deq = dequantize_blockwise(weights_quantized, absmax=absmax, code=code) + grad_input = grad_output @ weights_deq + grad_bias = grad_output.flatten(0, -2).sum(dim=0) if ctx._has_bias else None + return grad_input, None, None, None, grad_bias + + +class FrozenBNBEmbedding(nn.Module): + def __init__(self, weight, absmax, code): + super().__init__() + self.num_embeddings, self.embedding_dim = weight.shape + self.register_buffer("weight", weight.requires_grad_(False)) + self.register_buffer("absmax", absmax.requires_grad_(False)) + self.register_buffer("code", code.requires_grad_(False)) + self.adapter = None + + def forward(self, input, **kwargs): + with torch.no_grad(): + # note: both quantuized weights and input indices are *not* differentiable + weight_deq = dequantize_blockwise(self.weight, absmax=self.absmax, code=self.code) + output = F.embedding(input, weight_deq, **kwargs) + if self.adapter: + output += self.adapter(input) + return output + + @classmethod + def from_embedding(cls, embedding: nn.Embedding) -> "FrozenBNBEmbedding": + weights_int8, state = quantize_blockise_lowmemory(embedding.weight) + return cls(weights_int8, *state) + + def __repr__(self): + return f"{self.__class__.__name__}({self.num_embeddings}, {self.embedding_dim})" + + +def quantize_blockise_lowmemory(matrix: torch.Tensor, chunk_size: int = 2**20): + assert chunk_size % 4096 == 0 + code = None + chunks = [] + absmaxes = [] + flat_tensor = matrix.view(-1) + for i in range((matrix.numel() - 1) // chunk_size + 1): + input_chunk = flat_tensor[i * chunk_size : (i + 1) * chunk_size].clone() + quantized_chunk, (absmax_chunk, code) = quantize_blockwise(input_chunk, code=code) + chunks.append(quantized_chunk) + absmaxes.append(absmax_chunk) + + matrix_i8 = torch.cat(chunks).reshape_as(matrix) + absmax = torch.cat(absmaxes) + return matrix_i8, (absmax, code) + + +def convert_to_int8(model): + """Convert linear and embedding modules to 8-bit with optional adapters""" + for module in list(model.modules()): + for name, child in module.named_children(): + if isinstance(child, nn.Linear): + print(name, child) + setattr( + module, + name, + FrozenBNBLinear( + weight=torch.zeros(child.out_features, child.in_features, dtype=torch.uint8), + absmax=torch.zeros((child.weight.numel() - 1) // 4096 + 1), + code=torch.zeros(256), + bias=child.bias, + ), + ) + elif isinstance(child, nn.Embedding): + setattr( + module, + name, + FrozenBNBEmbedding( + weight=torch.zeros(child.num_embeddings, child.embedding_dim, dtype=torch.uint8), + absmax=torch.zeros((child.weight.numel() - 1) // 4096 + 1), + code=torch.zeros(256), + ), + ) + + +class GPTJBlock(transformers.models.gptj.modeling_gptj.GPTJBlock): + def __init__(self, config): + super().__init__(config) + + convert_to_int8(self.attn) + convert_to_int8(self.mlp) + + +class GPTJModel(transformers.models.gptj.modeling_gptj.GPTJModel): + def __init__(self, config): + super().__init__(config) + convert_to_int8(self) + + +class GPTJForCausalLM(transformers.models.gptj.modeling_gptj.GPTJForCausalLM): + def __init__(self, config): + super().__init__(config) + convert_to_int8(self) + +def add_adapters(model, adapter_dim=16): + assert adapter_dim > 0 + + for module in model.modules(): + if isinstance(module, FrozenBNBLinear): + module.adapter = nn.Sequential( + nn.Linear(module.in_features, adapter_dim, bias=False), + nn.Linear(adapter_dim, module.out_features, bias=False), + ) + nn.init.zeros_(module.adapter[1].weight) + elif isinstance(module, FrozenBNBEmbedding): + module.adapter = nn.Sequential( + nn.Embedding(module.num_embeddings, adapter_dim), + nn.Linear(adapter_dim, module.embedding_dim, bias=False), + ) + nn.init.zeros_(module.adapter[1].weight) + + +def get_model(model_name, cache_dir, quantization): + if quantization is None: + model = AutoModelForCausalLM.from_pretrained(model_name, cache_dir=cache_dir) + elif quantization == "8bit": + transformers.models.gptj.modeling_gptj.GPTJBlock = GPTJBlock + model = AutoModelForCausalLM.from_pretrained(model_name, cache_dir=cache_dir) + add_adapters(gpt) + else: + raise ValueError(f"Unknown quantization {quantization}") + + return model diff --git a/model/supervised_finetuning/trainer.py b/model/supervised_finetuning/trainer.py index dc7b5934..0db0f443 100644 --- a/model/supervised_finetuning/trainer.py +++ b/model/supervised_finetuning/trainer.py @@ -17,6 +17,7 @@ from transformers import ( TrainingArguments, get_cosine_schedule_with_warmup, ) +import bitsandbytes as bnb from utils import get_dataset, get_loss, get_model, get_tokenizer, read_yamls os.environ["WANDB_PROJECT"] = "supervised-finetuning" @@ -25,6 +26,7 @@ os.environ["WANDB_PROJECT"] = "supervised-finetuning" @dataclass class CustomTrainingArguments(TrainingArguments): loss_function: str = "CrossEntropyLoss" + quantization: str = None def compute_metrics(eval_pred): @@ -71,8 +73,16 @@ class SFTTrainer(Trainer): # By default CrossEntropyLoss ignores padding_index -100, but just in case use our own loss_fct self.loss_fct = get_loss(args.loss_function) - def fetch_scheduler(self): - return get_cosine_schedule_with_warmup( + def create_optimizer_and_scheduler(self, num_training_steps: int): + if self.args.quantization == "8bit": + self.optimizer = bnb.optim.Adam8bit(model.parameters(), lr=0.001, betas=(0.9, 0.995)) + else: + self.optimizer = torch.optim.AdamW( + self.model.parameters(), lr=self.args.learning_rate, weight_decay=self.args.weight_decay + ) + + print("lr sheduler") + self.lr_scheduler = get_cosine_schedule_with_warmup( self.optimizer, num_warmup_steps=self.args.warmup_steps, num_training_steps=self.num_train_steps, @@ -165,6 +175,7 @@ if __name__ == "__main__": model = get_model(training_conf, tokenizer) train, evals, collate_fn = get_dataset(training_conf, tokenizer) + assert len(evals) > 0 args = CustomTrainingArguments( output_dir=f"{training_conf.model_name}-{training_conf.log_dir}-finetuned", @@ -186,9 +197,9 @@ if __name__ == "__main__": save_steps=training_conf.save_steps, eval_accumulation_steps=training_conf.eval_accumulation_steps, report_to="wandb", + quantization=training_conf.quantization, ) - assert len(evals) > 0 trainer = SFTTrainer( model, args, diff --git a/model/supervised_finetuning/utils.py b/model/supervised_finetuning/utils.py index a31f74d3..318aeb5e 100644 --- a/model/supervised_finetuning/utils.py +++ b/model/supervised_finetuning/utils.py @@ -6,7 +6,8 @@ from custom_datasets.dialogue_collator import DialogueDataCollator from losses import CrossEntropyLoss from sklearn.model_selection import train_test_split from torch.utils.data import ConcatDataset, Subset -from transformers import AutoModelForCausalLM, AutoTokenizer +from transformers import AutoTokenizer +from models import get_specific_model SUPPORTED_MODELS = ["galactica", "GPT-JT"] # deprecated .. @@ -36,7 +37,7 @@ def get_model(conf, tokenizer): "To include more make sure the masking is dne correctly... (decoder only supported for now)" ) - model = AutoModelForCausalLM.from_pretrained(conf.model_name, cache_dir=conf.cache_dir) + model = get_specific_model(conf.model_name, conf.cache_dir, conf.quantization) if len(tokenizer) != model.get_input_embeddings().num_embeddings: assert not conf.freeze_layer, "Cannot change the number of embeddings if the model is frozen." From 5b9cdcec4870a1d651d6f18e56fd2fc34c26ac27 Mon Sep 17 00:00:00 2001 From: furlat Date: Thu, 5 Jan 2023 20:19:31 +0100 Subject: [PATCH 02/32] Create openbugger_example.ipynb added an example for open_bugger --- .../code-bugger/openbugger_example.ipynb | 273 ++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 notebooks/code-bugger/openbugger_example.ipynb diff --git a/notebooks/code-bugger/openbugger_example.ipynb b/notebooks/code-bugger/openbugger_example.ipynb new file mode 100644 index 00000000..2416e42a --- /dev/null +++ b/notebooks/code-bugger/openbugger_example.ipynb @@ -0,0 +1,273 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# First, we'll import the necessary libraries\n", + "import os\n", + "import subprocess" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## only run from OpenAssistant not if you alread cloned the github\n", + "# Next, we'll define the function that will clone the OpenBugger repository and install it.\n", + "def install_openbugger():\n", + " # First, we'll get the current working directory. This is where the repository will be cloned to.\n", + " cwd = os.getcwd()\n", + " \n", + " # Next, we'll use Git to clone the repository.\n", + " subprocess.run([\"git\", \"clone\", \"https://github.com/furlat/OpenBugger\", cwd + \"/OpenBugger\"])\n", + " \n", + " # Now, we'll use pip to install the package from the local repository.\n", + " subprocess.run([\"python3\", \"-m\", \"pip\", \"install\", \"--editable\", cwd + \"/OpenBugger\"])\n", + " \n", + "# Now, we'll call the function to install OpenBugger.\n", + "install_openbugger()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Finally, we'll import SyntaxBug and LogicBug.\n", + "from syntax.syntax_injector import SyntaxBug\n", + "from logic.logic_injector import LogicBug\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#setup syntax bug\n", + "syntax_bug = SyntaxBug()\n", + "\n", + "# Simple script\n", + "simple_script = \"\"\"\n", + "def greet(name):\n", + " print(\"Hello, \" + name)\n", + "\n", + "greet(\"Bob\")\n", + "\"\"\"\n", + "\n", + "# The simple script can be modified using the \"easy\" injection method because it only contains simple syntax and does not have any nested code blocks. This means that there are fewer characters (e.g. quotes, brackets, braces, parenthesis) that could be the target of syntax errors, and the \"easy\" injection method, which only injects errors that involve replacing or removing a single character, is sufficient to modify the script.\n", + "print(simple_script)\n", + "# Inject easy syntax errors into the simple script\n", + "\n", + "modified_simple_script, errors, counter = syntax_bug.inject(simple_script, \"easy\", 1)\n", + "print(\"Modified version Easy\",errors,counter)\n", + "print(modified_simple_script)\n", + "print(\"are they the same?\",simple_script == modified_simple_script)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Medium script\n", + "medium_script = \"\"\"\n", + "def greet(name):\n", + " print(\"Hello, \" + name)\n", + " \n", + "def greet_all(names):\n", + " for name in names:\n", + " greet(name)\n", + " \n", + "greet_all([\"Bob\", \"Alice\", \"Eve\"])\n", + "\"\"\"\n", + "#The medium script can be modified using the \"medium\" injection method because it contains a nested code block (the for loop in the `greet_all` function). This means that there are more characters (e.g. quotes, brackets, braces, parenthesis) that could be the target of syntax errors, and the \"medium\" injection method, which injects errors that involve replacing, removing, or adding a single character, is sufficient to modify the script.\n", + "print(medium_script)\n", + "# Inject medium syntax errors into the medium script\n", + "modified_medium_script, errors, counter = syntax_bug.inject(medium_script, \"medium\", 3)\n", + "print(\"Modified version Medium\",errors,counter)\n", + "print(modified_medium_script)\n", + "print(\"are they the same?\",medium_script == modified_medium_script)\n", + "# Hard script\n", + "hard_script = \"\"\"\n", + "class Greeting:\n", + " def __init__(self, greeting):\n", + " self.greeting = greeting\n", + " \n", + " def greet(self, name):\n", + " print(self.greeting + \", \" + name)\n", + " \n", + "greeting = Greeting(\"Hello\")\n", + "greeting.greet(\"Bob\")\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Hard script\n", + "hard_script = \"\"\"\n", + "class Greeting:\n", + " def __init__(self, greeting):\n", + " self.greeting = greeting\n", + " \n", + " def greet(self, name):\n", + " print(self.greeting + \", \" + name)\n", + " \n", + "greeting = Greeting(\"Hello\")\n", + "greeting.greet(\"Bob\")\n", + "\"\"\"\n", + "\n", + "# The hard script can be modified using the \"hard\" injection method because it contains multiple nested code blocks (the `__init__` and `greet` methods in the `Greeting` class). This means that there are even more characters (e.g. quotes, brackets, braces, parenthesis) that could be the target of syntax errors, and the \"hard\" injection method, which injects errors that involve replacing, removing, adding, or swapping characters, is sufficient to modify the script.\n", + "print(hard_script)\n", + "# Inject hard syntax errors into the hard script\n", + "modified_hard_script, errors, counter = syntax_bug.inject(hard_script, \"hard\", 3)\n", + "print(\"Modified version Hard\",errors,counter)\n", + "print(modified_hard_script)\n", + "print(\"are they the same?\",hard_script == modified_hard_script)\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we switch to testing the LogicBugs and show how to bug a function that is defined in the script without already having the string." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import inspect\n", + "import random\n", + "# Simple example script\n", + "\n", + "def simple_script():\n", + " # Choose two random integers\n", + " num1 = random.randint(0, 10)\n", + " num2 = random.randint(0, 10)\n", + " \n", + " # Compare the two numbers and print a message based on their relation\n", + " if num1 > num2:\n", + " print(\"num1 is greater than num2\")\n", + " elif num1 < num2:\n", + " print(\"num1 is less than num2\")\n", + " else:\n", + " print(\"num1 is equal to num2\")\n", + "\n", + "# Medium example script\n", + "def medium_script():\n", + " # Choose a random integer and assign it to a variable\n", + " num = random.randint(0, 10)\n", + " \n", + " # Use a loop to print all numbers from 0 to the chosen integer\n", + " for i in range(num):\n", + " print(i)\n", + "\n", + "# Hard example script\n", + "def hard_script():\n", + " # Choose a random integer and assign it to a variable\n", + " num = random.randint(0, 10)\n", + " \n", + " # Use a loop to print the square of all numbers from 0 to the chosen integer\n", + " for i in range(num):\n", + " print(i**2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# create an instance of the LogicBug class\n", + "logic_bug = LogicBug()\n", + "\n", + "# get the source code of the simple_script function as a string\n", + "simple_script_str = inspect.getsource(simple_script)\n", + "print(\"Simple\",simple_script_str)\n", + "# inject a logic error into the simple_script function\n", + "modified_simple_script, error, counter = logic_bug.inject(simple_script_str, \"easy\",num_errors=3)\n", + "print(\"Modified version Simple\",error,counter)\n", + "# print the modified simple_script function\n", + "print(modified_simple_script)\n", + "print(\"are they the same?\",simple_script_str == modified_simple_script)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get the source code of the medium_script function as a string\n", + "medium_script_str = inspect.getsource(medium_script)\n", + "print(\"Medium\",medium_script_str)\n", + "# inject a logic error into the medium_script function\n", + "modified_medium_script, error, counter = logic_bug.inject(medium_script_str,\"medium\",num_errors=3)\n", + "\n", + "# print the modified medium_script function\n", + "print(\"Modified version Medium\",error,counter)\n", + "print(modified_medium_script)\n", + "print(\"are they the same?\",medium_script_str == modified_medium_script)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get the source code of the hard_script function as a string\n", + "hard_script_str = inspect.getsource(hard_script)\n", + "print(\"Hard\",hard_script_str)\n", + "# inject a logic error into the hard_script function\n", + "modified_hard_script, error, counter = logic_bug.inject(hard_script_str,\"hard\",num_errors=1)\n", + "print(\"Modified version Hard\",error,counter)\n", + "# print the modified hard_script function\n", + "print(modified_hard_script)\n", + "print(\"are they the same?\",hard_script_str == modified_hard_script)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2 (default, Apr 8 2021, 23:19:18) \n[Clang 12.0.5 (clang-1205.0.22.9)]" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 51739fa5b72d08e4cbc21ca81a17a0bb5e54542f Mon Sep 17 00:00:00 2001 From: furlat Date: Thu, 5 Jan 2023 20:33:04 +0100 Subject: [PATCH 03/32] Update openbugger_example.ipynb --- .../code-bugger/openbugger_example.ipynb | 61 ++++++++++--------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/notebooks/code-bugger/openbugger_example.ipynb b/notebooks/code-bugger/openbugger_example.ipynb index 2416e42a..ec8583d1 100644 --- a/notebooks/code-bugger/openbugger_example.ipynb +++ b/notebooks/code-bugger/openbugger_example.ipynb @@ -22,15 +22,16 @@ "def install_openbugger():\n", " # First, we'll get the current working directory. This is where the repository will be cloned to.\n", " cwd = os.getcwd()\n", - " \n", + "\n", " # Next, we'll use Git to clone the repository.\n", " subprocess.run([\"git\", \"clone\", \"https://github.com/furlat/OpenBugger\", cwd + \"/OpenBugger\"])\n", - " \n", + "\n", " # Now, we'll use pip to install the package from the local repository.\n", " subprocess.run([\"python3\", \"-m\", \"pip\", \"install\", \"--editable\", cwd + \"/OpenBugger\"])\n", - " \n", + "\n", + "\n", "# Now, we'll call the function to install OpenBugger.\n", - "install_openbugger()\n" + "install_openbugger()" ] }, { @@ -41,7 +42,7 @@ "source": [ "# Finally, we'll import SyntaxBug and LogicBug.\n", "from syntax.syntax_injector import SyntaxBug\n", - "from logic.logic_injector import LogicBug\n" + "from logic.logic_injector import LogicBug" ] }, { @@ -50,7 +51,7 @@ "metadata": {}, "outputs": [], "source": [ - "#setup syntax bug\n", + "# setup syntax bug\n", "syntax_bug = SyntaxBug()\n", "\n", "# Simple script\n", @@ -66,9 +67,9 @@ "# Inject easy syntax errors into the simple script\n", "\n", "modified_simple_script, errors, counter = syntax_bug.inject(simple_script, \"easy\", 1)\n", - "print(\"Modified version Easy\",errors,counter)\n", + "print(\"Modified version Easy\", errors, counter)\n", "print(modified_simple_script)\n", - "print(\"are they the same?\",simple_script == modified_simple_script)" + "print(\"are they the same?\", simple_script == modified_simple_script)" ] }, { @@ -88,13 +89,13 @@ " \n", "greet_all([\"Bob\", \"Alice\", \"Eve\"])\n", "\"\"\"\n", - "#The medium script can be modified using the \"medium\" injection method because it contains a nested code block (the for loop in the `greet_all` function). This means that there are more characters (e.g. quotes, brackets, braces, parenthesis) that could be the target of syntax errors, and the \"medium\" injection method, which injects errors that involve replacing, removing, or adding a single character, is sufficient to modify the script.\n", + "# The medium script can be modified using the \"medium\" injection method because it contains a nested code block (the for loop in the `greet_all` function). This means that there are more characters (e.g. quotes, brackets, braces, parenthesis) that could be the target of syntax errors, and the \"medium\" injection method, which injects errors that involve replacing, removing, or adding a single character, is sufficient to modify the script.\n", "print(medium_script)\n", "# Inject medium syntax errors into the medium script\n", "modified_medium_script, errors, counter = syntax_bug.inject(medium_script, \"medium\", 3)\n", - "print(\"Modified version Medium\",errors,counter)\n", + "print(\"Modified version Medium\", errors, counter)\n", "print(modified_medium_script)\n", - "print(\"are they the same?\",medium_script == modified_medium_script)\n", + "print(\"are they the same?\", medium_script == modified_medium_script)\n", "# Hard script\n", "hard_script = \"\"\"\n", "class Greeting:\n", @@ -132,9 +133,9 @@ "print(hard_script)\n", "# Inject hard syntax errors into the hard script\n", "modified_hard_script, errors, counter = syntax_bug.inject(hard_script, \"hard\", 3)\n", - "print(\"Modified version Hard\",errors,counter)\n", + "print(\"Modified version Hard\", errors, counter)\n", "print(modified_hard_script)\n", - "print(\"are they the same?\",hard_script == modified_hard_script)\n" + "print(\"are they the same?\", hard_script == modified_hard_script)" ] }, { @@ -153,13 +154,15 @@ "source": [ "import inspect\n", "import random\n", + "\n", "# Simple example script\n", "\n", + "\n", "def simple_script():\n", " # Choose two random integers\n", " num1 = random.randint(0, 10)\n", " num2 = random.randint(0, 10)\n", - " \n", + "\n", " # Compare the two numbers and print a message based on their relation\n", " if num1 > num2:\n", " print(\"num1 is greater than num2\")\n", @@ -168,20 +171,22 @@ " else:\n", " print(\"num1 is equal to num2\")\n", "\n", + "\n", "# Medium example script\n", "def medium_script():\n", " # Choose a random integer and assign it to a variable\n", " num = random.randint(0, 10)\n", - " \n", + "\n", " # Use a loop to print all numbers from 0 to the chosen integer\n", " for i in range(num):\n", " print(i)\n", "\n", + "\n", "# Hard example script\n", "def hard_script():\n", " # Choose a random integer and assign it to a variable\n", " num = random.randint(0, 10)\n", - " \n", + "\n", " # Use a loop to print the square of all numbers from 0 to the chosen integer\n", " for i in range(num):\n", " print(i**2)" @@ -198,13 +203,13 @@ "\n", "# get the source code of the simple_script function as a string\n", "simple_script_str = inspect.getsource(simple_script)\n", - "print(\"Simple\",simple_script_str)\n", + "print(\"Simple\", simple_script_str)\n", "# inject a logic error into the simple_script function\n", - "modified_simple_script, error, counter = logic_bug.inject(simple_script_str, \"easy\",num_errors=3)\n", - "print(\"Modified version Simple\",error,counter)\n", + "modified_simple_script, error, counter = logic_bug.inject(simple_script_str, \"easy\", num_errors=3)\n", + "print(\"Modified version Simple\", error, counter)\n", "# print the modified simple_script function\n", "print(modified_simple_script)\n", - "print(\"are they the same?\",simple_script_str == modified_simple_script)\n" + "print(\"are they the same?\", simple_script_str == modified_simple_script)" ] }, { @@ -215,14 +220,14 @@ "source": [ "# get the source code of the medium_script function as a string\n", "medium_script_str = inspect.getsource(medium_script)\n", - "print(\"Medium\",medium_script_str)\n", + "print(\"Medium\", medium_script_str)\n", "# inject a logic error into the medium_script function\n", - "modified_medium_script, error, counter = logic_bug.inject(medium_script_str,\"medium\",num_errors=3)\n", + "modified_medium_script, error, counter = logic_bug.inject(medium_script_str, \"medium\", num_errors=3)\n", "\n", "# print the modified medium_script function\n", - "print(\"Modified version Medium\",error,counter)\n", + "print(\"Modified version Medium\", error, counter)\n", "print(modified_medium_script)\n", - "print(\"are they the same?\",medium_script_str == modified_medium_script)" + "print(\"are they the same?\", medium_script_str == modified_medium_script)" ] }, { @@ -233,13 +238,13 @@ "source": [ "# get the source code of the hard_script function as a string\n", "hard_script_str = inspect.getsource(hard_script)\n", - "print(\"Hard\",hard_script_str)\n", + "print(\"Hard\", hard_script_str)\n", "# inject a logic error into the hard_script function\n", - "modified_hard_script, error, counter = logic_bug.inject(hard_script_str,\"hard\",num_errors=1)\n", - "print(\"Modified version Hard\",error,counter)\n", + "modified_hard_script, error, counter = logic_bug.inject(hard_script_str, \"hard\", num_errors=1)\n", + "print(\"Modified version Hard\", error, counter)\n", "# print the modified hard_script function\n", "print(modified_hard_script)\n", - "print(\"are they the same?\",hard_script_str == modified_hard_script)\n" + "print(\"are they the same?\", hard_script_str == modified_hard_script)" ] } ], From bb66f5d750622aa10d6431acfa425a92addaf3b3 Mon Sep 17 00:00:00 2001 From: furlat Date: Thu, 5 Jan 2023 23:09:01 +0100 Subject: [PATCH 04/32] Added md for openbugger_example --- .../code-bugger/openbugger_example.ipynb | 4 +- notebooks/code-bugger/openbugger_example.md | 70 +++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 notebooks/code-bugger/openbugger_example.md diff --git a/notebooks/code-bugger/openbugger_example.ipynb b/notebooks/code-bugger/openbugger_example.ipynb index ec8583d1..22e9b0c8 100644 --- a/notebooks/code-bugger/openbugger_example.ipynb +++ b/notebooks/code-bugger/openbugger_example.ipynb @@ -264,12 +264,12 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.2 (default, Apr 8 2021, 23:19:18) \n[Clang 12.0.5 (clang-1205.0.22.9)]" + "version": "3.10.6 (tags/v3.10.6:9c7b4bd, Aug 1 2022, 21:53:49) [MSC v.1932 64 bit (AMD64)]" }, "orig_nbformat": 4, "vscode": { "interpreter": { - "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + "hash": "ceba285e8b4e6478fe8ad229bc63940a90ad5cf3d143521e7c38823a2e915b21" } } }, diff --git a/notebooks/code-bugger/openbugger_example.md b/notebooks/code-bugger/openbugger_example.md new file mode 100644 index 00000000..0ce05de7 --- /dev/null +++ b/notebooks/code-bugger/openbugger_example.md @@ -0,0 +1,70 @@ +# OpenBuggerNotebook +https://github.com/furlat/OpenBugger/blob/main/README.md is a Python package that allows you to inject syntax and logic errors into your code. This can be useful for testing the robustness of your code or for creating test cases for debugging exercises or for training an assistant to debug code. + +The Python notebook openbugger_example.ipynb does the following: + +1) Imports the necessary libraries to install OpenBugger in the notebookdirecory (os and subprocess). +2) Defines a function, install_openbugger, which clones the OpenBugger repository from GitHub and installs it using pip. +3) Calls the install_openbugger function to install OpenBugger. +5) Imports the SyntaxBug and LogicBug classes from the syntax_injector and logic_injector modules, respectively. +6) Creates an instance of the SyntaxBug class and assigns it to the syntax_bug variable. +7) Defines three scripts: a simple script, a medium script, and a hard script. +8) Calls the inject method on the simple script, passing in the string "easy" as the second argument and the integer 1 as the third argument. This will inject easy syntax errors into the script. The *item *item modified script, a list of the injected errors, and the number of errors injected are returned and assigned to variables. +9) Prints the original and modified versions of the simple script, as well as the list of injected errors and the number of errors injected. +10 Repeats steps 7 and 8 for the medium and hard scripts, but with the "medium" and "hard" injection methods and different numbers of errors to inject. + + +General Usage +To use OpenBugger, import the SintaxBug or LogicBug classes from the openbugger module and use them to inject a bug with a call to the inject(). The injector will return the modified script with the injected bug. + +``` +from syntax.syntax_injector import SyntaxInjector, SyntaxBug + +syntax_bug = SyntaxBug() + + + +# Simple script +simple_script = """ +def greet(name): + print("Hello, " + name) + +greet("Bob") +""" + +print(simple_script) +``` +The simple script can be modified using the "easy" injection method because it only contains simple syntax and does not have any nested code blocks. +This means that there are fewer characters (e.g. quotes, brackets, braces, parenthesis) that could be the target of syntax errors, + and the "easy" injection method, which only injects errors that involve replacing or removing a single character, is sufficient to modify the script. +``` +# Inject easy syntax errors into the simple script + +modified_simple_script, errors, counter = syntax_bug.inject(simple_script, "easy", 1) +print("Modified version Easy",errors,counter) +print(modified_simple_script) +``` +Or for higher severity and logic error by directly transforming a Python class into text +``` +import inspect +import random +from logic.logic_injector import LogicBug + + +# Medium example script +def medium_script(): + # Choose a random integer and assign it to a variable + num = random.randint(0, 10) + + # Use a loop to print all numbers from 0 to the chosen integer + for i in range(num): + print(i) + +# create an instance of the LogicBug class +logic_bug = LogicBug() +# get the source code of the medium_script function as a string +medium_script_str = inspect.getsource(medium_script) +print("Medium",medium_script_str) +# inject a logic error into the medium_script function +modified_medium_script, error, counter = logic_bug.inject(medium_script_str,"medium",num_errors=3) +``` \ No newline at end of file From f7ee8fd74bd4da2198e286b769c3466101845f1c Mon Sep 17 00:00:00 2001 From: furlat Date: Thu, 5 Jan 2023 23:10:34 +0100 Subject: [PATCH 05/32] pre_commit openbugger md --- notebooks/code-bugger/openbugger_example.md | 62 ++++++++++++++------- 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/notebooks/code-bugger/openbugger_example.md b/notebooks/code-bugger/openbugger_example.md index 0ce05de7..d8611dd7 100644 --- a/notebooks/code-bugger/openbugger_example.md +++ b/notebooks/code-bugger/openbugger_example.md @@ -1,21 +1,35 @@ # OpenBuggerNotebook -https://github.com/furlat/OpenBugger/blob/main/README.md is a Python package that allows you to inject syntax and logic errors into your code. This can be useful for testing the robustness of your code or for creating test cases for debugging exercises or for training an assistant to debug code. + +https://github.com/furlat/OpenBugger/blob/main/README.md is a Python package +that allows you to inject syntax and logic errors into your code. This can be +useful for testing the robustness of your code or for creating test cases for +debugging exercises or for training an assistant to debug code. The Python notebook openbugger_example.ipynb does the following: -1) Imports the necessary libraries to install OpenBugger in the notebookdirecory (os and subprocess). -2) Defines a function, install_openbugger, which clones the OpenBugger repository from GitHub and installs it using pip. -3) Calls the install_openbugger function to install OpenBugger. -5) Imports the SyntaxBug and LogicBug classes from the syntax_injector and logic_injector modules, respectively. -6) Creates an instance of the SyntaxBug class and assigns it to the syntax_bug variable. -7) Defines three scripts: a simple script, a medium script, and a hard script. -8) Calls the inject method on the simple script, passing in the string "easy" as the second argument and the integer 1 as the third argument. This will inject easy syntax errors into the script. The *item *item modified script, a list of the injected errors, and the number of errors injected are returned and assigned to variables. -9) Prints the original and modified versions of the simple script, as well as the list of injected errors and the number of errors injected. -10 Repeats steps 7 and 8 for the medium and hard scripts, but with the "medium" and "hard" injection methods and different numbers of errors to inject. +1. Imports the necessary libraries to install OpenBugger in the notebookdirecory + (os and subprocess). +2. Defines a function, install_openbugger, which clones the OpenBugger + repository from GitHub and installs it using pip. +3. Calls the install_openbugger function to install OpenBugger. +4. Imports the SyntaxBug and LogicBug classes from the syntax_injector and + logic_injector modules, respectively. +5. Creates an instance of the SyntaxBug class and assigns it to the syntax_bug + variable. +6. Defines three scripts: a simple script, a medium script, and a hard script. +7. Calls the inject method on the simple script, passing in the string "easy" as + the second argument and the integer 1 as the third argument. This will inject + easy syntax errors into the script. The *item *item modified script, a list + of the injected errors, and the number of errors injected are returned and + assigned to variables. +8. Prints the original and modified versions of the simple script, as well as + the list of injected errors and the number of errors injected. 10 Repeats + steps 7 and 8 for the medium and hard scripts, but with the "medium" and + "hard" injection methods and different numbers of errors to inject. - -General Usage -To use OpenBugger, import the SintaxBug or LogicBug classes from the openbugger module and use them to inject a bug with a call to the inject(). The injector will return the modified script with the injected bug. +General Usage To use OpenBugger, import the SintaxBug or LogicBug classes from +the openbugger module and use them to inject a bug with a call to the inject(). +The injector will return the modified script with the injected bug. ``` from syntax.syntax_injector import SyntaxInjector, SyntaxBug @@ -31,12 +45,17 @@ def greet(name): greet("Bob") """ - + print(simple_script) ``` -The simple script can be modified using the "easy" injection method because it only contains simple syntax and does not have any nested code blocks. -This means that there are fewer characters (e.g. quotes, brackets, braces, parenthesis) that could be the target of syntax errors, - and the "easy" injection method, which only injects errors that involve replacing or removing a single character, is sufficient to modify the script. + +The simple script can be modified using the "easy" injection method because it +only contains simple syntax and does not have any nested code blocks. This means +that there are fewer characters (e.g. quotes, brackets, braces, parenthesis) +that could be the target of syntax errors, and the "easy" injection method, +which only injects errors that involve replacing or removing a single character, +is sufficient to modify the script. + ``` # Inject easy syntax errors into the simple script @@ -44,7 +63,10 @@ modified_simple_script, errors, counter = syntax_bug.inject(simple_script, "easy print("Modified version Easy",errors,counter) print(modified_simple_script) ``` -Or for higher severity and logic error by directly transforming a Python class into text + +Or for higher severity and logic error by directly transforming a Python class +into text + ``` import inspect import random @@ -55,7 +77,7 @@ from logic.logic_injector import LogicBug def medium_script(): # Choose a random integer and assign it to a variable num = random.randint(0, 10) - + # Use a loop to print all numbers from 0 to the chosen integer for i in range(num): print(i) @@ -67,4 +89,4 @@ medium_script_str = inspect.getsource(medium_script) print("Medium",medium_script_str) # inject a logic error into the medium_script function modified_medium_script, error, counter = logic_bug.inject(medium_script_str,"medium",num_errors=3) -``` \ No newline at end of file +``` From a321c53ac637012de7e1341f8a8b3e0d5bc3384f Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 6 Jan 2023 11:55:59 +0000 Subject: [PATCH 06/32] fix hrefs on readme badges my bad --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a443a5d7..e6a682ec 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,11 @@
-![GitHub Repo stars](https://img.shields.io/github/stars/LAION-AI/Open-Assistant?style=social) -![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/LAION-AI/Open-Assistant/build-frontend.yaml?label=frontend) -![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/LAION-AI/Open-Assistant/pre-commit.yaml?label=pre-commit) -![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/LAION-AI/Open-Assistant/test-api-contract.yaml?label=api) -![GitHub release (latest by date)](https://img.shields.io/github/v/release/LAION-AI/Open-Assistant) +![GitHub Repo stars](https://img.shields.io/github/stars/LAION-AI/Open-Assistant?style=social) +![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/LAION-AI/Open-Assistant/build-frontend.yaml?label=frontend) +![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/LAION-AI/Open-Assistant/pre-commit.yaml?label=pre-commit) +![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/LAION-AI/Open-Assistant/test-api-contract.yaml?label=api) +![GitHub release (latest by date)](https://img.shields.io/github/v/release/LAION-AI/Open-Assistant)
From 21986694c97a76b4e98b9b59f862424bc45d87d9 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 6 Jan 2023 13:16:46 +0000 Subject: [PATCH 07/32] try multiple folders --- .devcontainer/{ => default}/devcontainer.json | 0 .../{ => default}/post_create_command.sh | 0 .devcontainer/frontend/devcontainer.json | 15 +++++++++++++++ .devcontainer/frontend/post_create_command.sh | 2 ++ 4 files changed, 17 insertions(+) rename .devcontainer/{ => default}/devcontainer.json (100%) rename .devcontainer/{ => default}/post_create_command.sh (100%) create mode 100644 .devcontainer/frontend/devcontainer.json create mode 100644 .devcontainer/frontend/post_create_command.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/default/devcontainer.json similarity index 100% rename from .devcontainer/devcontainer.json rename to .devcontainer/default/devcontainer.json diff --git a/.devcontainer/post_create_command.sh b/.devcontainer/default/post_create_command.sh similarity index 100% rename from .devcontainer/post_create_command.sh rename to .devcontainer/default/post_create_command.sh diff --git a/.devcontainer/frontend/devcontainer.json b/.devcontainer/frontend/devcontainer.json new file mode 100644 index 00000000..22f43374 --- /dev/null +++ b/.devcontainer/frontend/devcontainer.json @@ -0,0 +1,15 @@ +{ + "name": "Open-Assistant", + "image": "mcr.microsoft.com/vscode/devcontainers/universal", + "features": { + "ghcr.io/devcontainers-contrib/features/pre-commit:2": { + "version": "latest" + } + }, + "postCreateCommand": "bash .devcontainer/post_create_command.sh", + "customizations": { + "vscode": { + "extensions": ["GitHub.copilot"] + } + } +} diff --git a/.devcontainer/frontend/post_create_command.sh b/.devcontainer/frontend/post_create_command.sh new file mode 100644 index 00000000..983576b9 --- /dev/null +++ b/.devcontainer/frontend/post_create_command.sh @@ -0,0 +1,2 @@ +# ensure pre-commit is installed +pre-commit install From cbd99e1ec8f01ed08a0cb602cb60eab6cb51daa1 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 6 Jan 2023 13:41:46 +0000 Subject: [PATCH 08/32] try subfolder --- .devcontainer/{default => }/devcontainer.json | 0 .devcontainer/frontend/post_create_command.sh | 5 +++++ .devcontainer/{default => }/post_create_command.sh | 0 3 files changed, 5 insertions(+) rename .devcontainer/{default => }/devcontainer.json (100%) rename .devcontainer/{default => }/post_create_command.sh (100%) diff --git a/.devcontainer/default/devcontainer.json b/.devcontainer/devcontainer.json similarity index 100% rename from .devcontainer/default/devcontainer.json rename to .devcontainer/devcontainer.json diff --git a/.devcontainer/frontend/post_create_command.sh b/.devcontainer/frontend/post_create_command.sh index 983576b9..001ea00e 100644 --- a/.devcontainer/frontend/post_create_command.sh +++ b/.devcontainer/frontend/post_create_command.sh @@ -1,2 +1,7 @@ # ensure pre-commit is installed pre-commit install + +# npm install in /website +cd website +npm install +cd .. \ No newline at end of file diff --git a/.devcontainer/default/post_create_command.sh b/.devcontainer/post_create_command.sh similarity index 100% rename from .devcontainer/default/post_create_command.sh rename to .devcontainer/post_create_command.sh From b4cbbe13b0130b9864816fc9c37fa6aeac9ef5c5 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 6 Jan 2023 13:56:45 +0000 Subject: [PATCH 09/32] add `frontend` devcontainer --- .devcontainer/README.md | 2 ++ .devcontainer/frontend/devcontainer.json | 2 +- .devcontainer/frontend/post_create_command.sh | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.devcontainer/README.md b/.devcontainer/README.md index a7e792da..9be25ac7 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -8,6 +8,8 @@ devcontainer (either or in your browser via [GitHub Codespaces](https://github.com/features/codespaces)). +**Note**: If you want to chose a specific .devcontainer within GitHub codespaces select "New with options" and you will be able to select any of the pre-defined devcontainers in this repo. + ### Run pre-commit ```bash diff --git a/.devcontainer/frontend/devcontainer.json b/.devcontainer/frontend/devcontainer.json index 22f43374..4f3c61ce 100644 --- a/.devcontainer/frontend/devcontainer.json +++ b/.devcontainer/frontend/devcontainer.json @@ -6,7 +6,7 @@ "version": "latest" } }, - "postCreateCommand": "bash .devcontainer/post_create_command.sh", + "postCreateCommand": "bash .devcontainer/frontend/post_create_command.sh", "customizations": { "vscode": { "extensions": ["GitHub.copilot"] diff --git a/.devcontainer/frontend/post_create_command.sh b/.devcontainer/frontend/post_create_command.sh index 001ea00e..2e5af4e3 100644 --- a/.devcontainer/frontend/post_create_command.sh +++ b/.devcontainer/frontend/post_create_command.sh @@ -4,4 +4,4 @@ pre-commit install # npm install in /website cd website npm install -cd .. \ No newline at end of file +cd .. From c0b2676d7961d2fa3ee23ac7291f1a191d92b732 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 6 Jan 2023 14:02:19 +0000 Subject: [PATCH 10/32] fix pre-commit --- .devcontainer/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 9be25ac7..c93a9660 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -8,7 +8,9 @@ devcontainer (either or in your browser via [GitHub Codespaces](https://github.com/features/codespaces)). -**Note**: If you want to chose a specific .devcontainer within GitHub codespaces select "New with options" and you will be able to select any of the pre-defined devcontainers in this repo. +**Note**: If you want to chose a specific .devcontainer within GitHub codespaces +select "New with options" and you will be able to select any of the pre-defined +devcontainers in this repo. ### Run pre-commit From 667d51df55cdadd8178f13abf40d14ef20d27d13 Mon Sep 17 00:00:00 2001 From: chs20 Date: Fri, 6 Jan 2023 15:30:20 +0100 Subject: [PATCH 11/32] Fix message flagging in dark mode (#434) --- website/src/components/FlaggableElement.tsx | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/website/src/components/FlaggableElement.tsx b/website/src/components/FlaggableElement.tsx index bc245bd2..a55d29c5 100644 --- a/website/src/components/FlaggableElement.tsx +++ b/website/src/components/FlaggableElement.tsx @@ -17,12 +17,15 @@ import { Spacer, Tooltip, useBoolean, + useColorMode, + useColorModeValue, useId, } from "@chakra-ui/react"; import { FlagIcon, QuestionMarkCircleIcon } from "@heroicons/react/20/solid"; import { useState } from "react"; import poster from "src/lib/poster"; import useSWRMutation from "swr/mutation"; +import { colors } from "styles/Theme/colors"; export const FlaggableElement = (props) => { const [isEditing, setIsEditing] = useBoolean(); @@ -102,7 +105,10 @@ export const FlaggableElement = (props) => { @@ -133,6 +139,12 @@ function FlagCheckbox(props: { } const id = useId(); + const { colorMode } = useColorMode(); + + const labelTextClass = + colorMode === "light" + ? `text-${colors.light.text} hover:text-blue-700 float-left` + : `text-${colors.dark.text} hover:text-blue-400 float-left`; return ( @@ -143,7 +155,7 @@ function FlagCheckbox(props: { }} /> From 083e8a869b05b7712b66dd047b4b98a66957b665 Mon Sep 17 00:00:00 2001 From: Jack Michaud Date: Fri, 6 Jan 2023 10:13:54 -0500 Subject: [PATCH 12/32] test: add unittest tests for edge cases in OasstApiClient Fixes #354 --- .../contract/oasst_api_contract_tests.cy.ts | 69 +++++++++++++++++-- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/website/cypress/contract/oasst_api_contract_tests.cy.ts b/website/cypress/contract/oasst_api_contract_tests.cy.ts index ff5bb156..2abbdf41 100644 --- a/website/cypress/contract/oasst_api_contract_tests.cy.ts +++ b/website/cypress/contract/oasst_api_contract_tests.cy.ts @@ -1,4 +1,4 @@ -import OasstApiClient from "src/lib/oasst_api_client"; +import OasstApiClient, { OasstError } from "src/lib/oasst_api_client"; describe("Contract test for Oasst API", function () { // Assumes this is running the mock server. @@ -23,7 +23,68 @@ describe("Contract test for Oasst API", function () { expect(await oasstApiClient.ackTask(task.id, "321")).to.be.null; }); - // TODO(#354): Add test for 204 - // TODO(#354): Add test for parsing >=300, throwing an OasstError - // TODO(#354): Add test for parsing >=300, throwing a generic error + // Note: Below here are unittests for OasstApiClient, not contract tests. They still fit here because + // the other contract tests are also testing the OasstApiClient. + const callApiMethod = () => { + return oasstApiClient.ackTask("", ""); + }; + + it("should return null for a 204 response", async () => { + const mockFetch = cy.stub(global, "fetch").resolves({ + status: 204, + }); + + const result = await callApiMethod(); + assert.isNull(result); + + mockFetch.restore(); + }); + + it("should throw an OasstError with data from the response for a non-2XX response with a valid OasstError response", async () => { + const mockFetch = cy.stub(global, "fetch").resolves({ + status: 400, + text: () => + // Note: this is vulnerable to interface changes in the Oasst API. + // The python tests use a Pydantic model to ensure this object is always valid, + // but we don't have that here. + // This could be a case for generating Zod schemas from OpenAPI. + Promise.resolve( + JSON.stringify({ + message: "error message", + error_code: 1000, + }) + ), + }); + + try { + await callApiMethod(); + assert.fail(); + } catch (error) { + assert.instanceOf(error, OasstError); + if (error instanceof OasstError) { + assert.equal(error.errorCode, 1000); + assert.equal(error.message, "error message"); + assert.equal(error.httpStatusCode, 400); + } + } + + mockFetch.restore(); + }); + + it("should throw a generic OasstError with the text from the response for a non-2XX response with an unknown format", async () => { + const mockFetch = cy.stub(global, "fetch").resolves({ + status: 400, + text: () => Promise.resolve("error message"), + }); + + try { + await callApiMethod(); + assert.fail(); + } catch (error) { + assert.instanceOf(error, OasstError); + assert.equal(error.message, "error message"); + } + + mockFetch.restore(); + }); }); From a9a3fc4e4ac6123ff31ee9c83e7dd0420a7c5ff6 Mon Sep 17 00:00:00 2001 From: Jack Michaud Date: Fri, 6 Jan 2023 10:15:07 -0500 Subject: [PATCH 13/32] fix: handle OasstError correctly in OasstApiClient --- website/src/lib/oasst_api_client.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/website/src/lib/oasst_api_client.ts b/website/src/lib/oasst_api_client.ts index 45a0859e..8b2ade72 100644 --- a/website/src/lib/oasst_api_client.ts +++ b/website/src/lib/oasst_api_client.ts @@ -1,6 +1,6 @@ import { JWT } from "next-auth/jwt"; -class OasstError { +export class OasstError { message: string; errorCode: number; httpStatusCode: number; @@ -31,12 +31,13 @@ export default class OasstApiClient { if (resp.status >= 300) { const errorText = await resp.text(); + let error: any; try { - const error = JSON.parse(errorText); - throw new OasstError(error.message, error.error_code, resp.status); + error = JSON.parse(errorText); } catch (e) { throw new OasstError(errorText, 0, resp.status); } + throw new OasstError(error.message, error.error_code, resp.status); } return await resp.json(); From 239c10c2dc02ac6bd32867cf5f2233813604ae69 Mon Sep 17 00:00:00 2001 From: Jack Michaud Date: Fri, 6 Jan 2023 10:21:23 -0500 Subject: [PATCH 14/32] fix: export OasstError --- website/cypress/contract/oasst_api_contract_tests.cy.ts | 2 +- website/src/lib/oasst_api_client.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/website/cypress/contract/oasst_api_contract_tests.cy.ts b/website/cypress/contract/oasst_api_contract_tests.cy.ts index 113e5e9c..47d43c34 100644 --- a/website/cypress/contract/oasst_api_contract_tests.cy.ts +++ b/website/cypress/contract/oasst_api_contract_tests.cy.ts @@ -1,4 +1,4 @@ -import { OasstApiClient } from "src/lib/oasst_api_client"; +import { OasstApiClient, OasstError } from "src/lib/oasst_api_client"; describe("Contract test for Oasst API", function () { // Assumes this is running the mock server. diff --git a/website/src/lib/oasst_api_client.ts b/website/src/lib/oasst_api_client.ts index 3755d676..4cf891e1 100644 --- a/website/src/lib/oasst_api_client.ts +++ b/website/src/lib/oasst_api_client.ts @@ -5,7 +5,7 @@ declare global { var oasstApiClient: OasstApiClient | undefined; } -class OasstError { +export class OasstError { message: string; errorCode: number; httpStatusCode: number; From b67181776a99cdeaac3de52e0a8ce7d543c2de0d Mon Sep 17 00:00:00 2001 From: theblackcat102 Date: Fri, 6 Jan 2023 16:47:58 +0000 Subject: [PATCH 15/32] [feature] add deepspeed, rallio dialogue dataset and codegen parameters --- .../supervised_finetuning/configs/config.yaml | 14 +++- .../configs/zero_config.json | 52 +++++++++++++ .../custom_datasets/__init__.py | 5 ++ .../custom_datasets/dialogue_collator.py | 1 + .../custom_datasets/prompt_dialogue.py | 77 +++++++++++++++++++ model/supervised_finetuning/losses.py | 2 +- model/supervised_finetuning/trainer.py | 58 ++++---------- model/supervised_finetuning/utils.py | 11 +-- 8 files changed, 169 insertions(+), 51 deletions(-) create mode 100644 model/supervised_finetuning/configs/zero_config.json create mode 100644 model/supervised_finetuning/custom_datasets/prompt_dialogue.py diff --git a/model/supervised_finetuning/configs/config.yaml b/model/supervised_finetuning/configs/config.yaml index 29086395..2ceb1388 100644 --- a/model/supervised_finetuning/configs/config.yaml +++ b/model/supervised_finetuning/configs/config.yaml @@ -6,7 +6,7 @@ defaults: per_device_eval_batch_size: 2 weight_decay: 0.00 warmup_steps: 600 - eval_steps: 200 + eval_steps: 100 save_steps: 500 max_length: 512 num_train_epochs: 3 @@ -17,6 +17,7 @@ defaults: freeze_layer: datasets: - webgpt + - prompt_dialogue cache_dir: ~/.cache loss_fn: CrossEntropyLoss eval_size: @@ -43,6 +44,17 @@ gpt-jt: per_device_train_batch_size: 4 per_device_eval_batch_size: 4 +codegen: + learning_rate: 2e-6 + model_name: Salesforce/codegen-2B-multi + weight_decay: 0.01 + max_length: 812 + warmup_steps: 600 + gradient_checkpointing: false + gradient_accumulation_steps: 5 + per_device_train_batch_size: 4 + per_device_eval_batch_size: 4 + debug: eval_steps: 20 eval_size: 100 diff --git a/model/supervised_finetuning/configs/zero_config.json b/model/supervised_finetuning/configs/zero_config.json new file mode 100644 index 00000000..e196d6a0 --- /dev/null +++ b/model/supervised_finetuning/configs/zero_config.json @@ -0,0 +1,52 @@ +{ + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 0.1 + }, + "optimizer": { + "type": "AdamW", + "params": { + "lr": "auto", + "weight_decay": "auto" + } + }, + "scheduler": { + "type": "WarmupDecayLR", + "params": { + "warmup_min_lr": "auto", + "warmup_max_lr": "auto", + "warmup_num_steps": "auto", + "total_num_steps": "auto" + } + }, + "zero_optimization": { + "stage": 3, + "offload_optimizer": { + "device": "cpu", + "pin_memory": true + }, + "offload_param": { + "device": "cpu", + "pin_memory": true + }, + "overlap_comm": true, + "contiguous_gradients": true, + "reduce_bucket_size": "auto", + "stage3_prefetch_bucket_size": "auto", + "stage3_param_persistence_threshold": "auto", + "sub_group_size": 1e9, + "stage3_max_live_parameters": 1e9, + "stage3_max_reuse_distance": 1e9, + "stage3_gather_16bit_weights_on_model_save": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": 2.0, + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false +} diff --git a/model/supervised_finetuning/custom_datasets/__init__.py b/model/supervised_finetuning/custom_datasets/__init__.py index 7e3bdc79..60462186 100644 --- a/model/supervised_finetuning/custom_datasets/__init__.py +++ b/model/supervised_finetuning/custom_datasets/__init__.py @@ -2,6 +2,8 @@ from datasets import load_dataset from sklearn.model_selection import train_test_split from torch.utils.data import Dataset, Subset +from .prompt_dialogue import PromptGeneratedDataset + QA_SPECIAL_TOKENS = {"Question": "", "Answer": ""} @@ -63,6 +65,9 @@ def get_one_dataset(conf, dataset_name): elif dataset_name == "webgpt": dataset = WebGPT() train, eval = train_val_dataset(dataset, val_split=0.2) + elif dataset_name == "prompt_dialogue": + dataset = PromptGeneratedDataset() + train, eval = train_val_dataset(dataset, val_split=0.2) else: raise ValueError(f"Unknown dataset {dataset_name}") diff --git a/model/supervised_finetuning/custom_datasets/dialogue_collator.py b/model/supervised_finetuning/custom_datasets/dialogue_collator.py index f9e1bb5e..535b9007 100644 --- a/model/supervised_finetuning/custom_datasets/dialogue_collator.py +++ b/model/supervised_finetuning/custom_datasets/dialogue_collator.py @@ -81,6 +81,7 @@ class DialogueDataCollator: batch["label_masks"] = torch.stack([F.pad(torch.tensor(x), (0, dim - len(x))) for x in label_masks]) + # why the fuck? for k in list(batch.keys()): if k not in ["input_ids", "attention_mask", "label_masks"]: batch.pop(k) diff --git a/model/supervised_finetuning/custom_datasets/prompt_dialogue.py b/model/supervised_finetuning/custom_datasets/prompt_dialogue.py new file mode 100644 index 00000000..07f32266 --- /dev/null +++ b/model/supervised_finetuning/custom_datasets/prompt_dialogue.py @@ -0,0 +1,77 @@ +import os +from urllib.request import urlopen + +from torch.utils.data import Dataset + + +class PromptGeneratedDataset(Dataset): + """Generates from flan 11B + User: What are the best methods for preventing a slave trade? + + Rosey: The best methods .... + <|endoftext|> + + we are ignoring results with multiple lines for now + """ + + url = "https://github.com/Rallio67/language-model-agents/raw/main/chat_dialogue_v2_c.txt" + + def __init__(self) -> None: + super().__init__() + os.makedirs("datasets", exist_ok=True) + chat_dialogue = os.path.join("datasets", "chat_dialogue_v2_c.txt") + if not os.path.exists(chat_dialogue): + with urlopen(self.url) as file: + content = file.read().decode() + with open(chat_dialogue, "w") as fout: + fout.write(content) + + question = "" + answer = "" + self.pairs = [] + with open(chat_dialogue, "r") as f: + + for line in f: + try: + line = line.rstrip() + if len(line) == 0: + continue + elif line == "<|endoftext|>" and len(question) > 0 and len(answer) > 0: + self.pairs.append((question, answer)) + question = "" + answer = "" + elif line[:4] == "User": + question = line.split(":", maxsplit=1)[1] + elif line == "Rosey:": + question = "" + answer = "" + elif len(line) > 4: # should be bot answer + answer = line.split(":", maxsplit=1)[1] + except IndexError: + question = "" + answer = "" + + if len(question) > 0 and len(answer) > 0: + self.pairs.append((question, answer)) + + def __len__(self): + return len(self.pairs) + + def __getitem__(self, index): + question, answer = self.pairs[index] + return question, answer + + +if __name__ == "__main__": + from torch.utils.data import DataLoader + from transformers import AutoTokenizer + + from .dialogue_collator import DialogueDataCollator + + tokenizer = AutoTokenizer.from_pretrained("Salesforce/codegen-2B-multi") + tokenizer.add_special_tokens({"pad_token": "<|endoftext|>", "sep_token": "<|endoftext|>"}) + dataset = PromptGeneratedDataset() + collate_fn = DialogueDataCollator(tokenizer, padding=True, max_length=128) + dataloader = DataLoader(dataset, collate_fn=collate_fn, batch_size=5) + for batch in dataloader: + print(batch["input_ids"].shape) diff --git a/model/supervised_finetuning/losses.py b/model/supervised_finetuning/losses.py index 795396b9..0cc639cf 100644 --- a/model/supervised_finetuning/losses.py +++ b/model/supervised_finetuning/losses.py @@ -7,7 +7,7 @@ class CrossEntropyLoss(nn.CrossEntropyLoss): def forward(self, input, target, mask=None): if mask is not None: - mask = mask.view(-1) + mask = mask.view(-1).bool() input = input.view(-1, input.size(-1)) target = target.view(-1) input = input[mask] diff --git a/model/supervised_finetuning/trainer.py b/model/supervised_finetuning/trainer.py index dc7b5934..2e59edcc 100644 --- a/model/supervised_finetuning/trainer.py +++ b/model/supervised_finetuning/trainer.py @@ -1,32 +1,17 @@ import argparse import os -from dataclasses import dataclass from distutils.util import strtobool -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import torch from torch import nn from torch.utils.data import Dataset -from transformers import ( - DataCollator, - EvalPrediction, - PreTrainedModel, - PreTrainedTokenizerBase, - Trainer, - TrainerCallback, - TrainingArguments, - get_cosine_schedule_with_warmup, -) +from transformers import PreTrainedModel, Trainer, TrainingArguments, get_cosine_schedule_with_warmup from utils import get_dataset, get_loss, get_model, get_tokenizer, read_yamls os.environ["WANDB_PROJECT"] = "supervised-finetuning" -@dataclass -class CustomTrainingArguments(TrainingArguments): - loss_function: str = "CrossEntropyLoss" - - def compute_metrics(eval_pred): pred_ids = eval_pred.predictions labels = eval_pred.label_ids @@ -44,32 +29,13 @@ class SFTTrainer(Trainer): self, model: Union[PreTrainedModel, nn.Module] = None, args: TrainingArguments = None, - data_collator: Optional[DataCollator] = None, - train_dataset: Optional[Dataset] = None, - eval_dataset: Optional[Dataset] = None, - tokenizer: Optional[PreTrainedTokenizerBase] = None, - model_init: Callable[[], PreTrainedModel] = None, - compute_metrics: Optional[Callable[[EvalPrediction], Dict]] = None, - callbacks: Optional[List[TrainerCallback]] = None, - optimizers: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None), - preprocess_logits_for_metrics: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = None, + loss_function: str = "CrossEntropyLoss", + **kwargs, ): - super().__init__( - model, - args, - data_collator, - train_dataset, - eval_dataset, - tokenizer, - model_init, - compute_metrics, - callbacks, - optimizers, - preprocess_logits_for_metrics, - ) + super().__init__(model, args, **kwargs) # By default CrossEntropyLoss ignores padding_index -100, but just in case use our own loss_fct - self.loss_fct = get_loss(args.loss_function) + self.loss_fct = get_loss(loss_function) def fetch_scheduler(self): return get_cosine_schedule_with_warmup( @@ -131,6 +97,10 @@ def _strtobool(x): def argument_parsing(notebook=False, notebook_args=None): parser = argparse.ArgumentParser() parser.add_argument("--configs", nargs="+", required=True) + parser.add_argument("--local_rank", type=int, default=-1) + parser.add_argument("--deepspeed", action="store_true") + parser.add_argument("--no-deepspeed", dest="deepspeed", action="store_false") + parser.set_defaults(deepspeed=False) if notebook: args, remaining = parser.parse_known_args(notebook_args) @@ -147,6 +117,8 @@ def argument_parsing(notebook=False, notebook_args=None): else: conf.update(configs[name]) + conf["local_rank"] = args.local_rank + conf["deepspeed"] = args.deepspeed # Override config from command-line parser = argparse.ArgumentParser() for key, value in conf.items(): @@ -166,13 +138,14 @@ if __name__ == "__main__": train, evals, collate_fn = get_dataset(training_conf, tokenizer) - args = CustomTrainingArguments( + args = TrainingArguments( output_dir=f"{training_conf.model_name}-{training_conf.log_dir}-finetuned", num_train_epochs=training_conf.num_train_epochs, warmup_steps=training_conf.warmup_steps, - loss_function=training_conf.loss_fn, learning_rate=float(training_conf.learning_rate), + deepspeed="configs/zero_config.json" if training_conf.deepspeed else None, fp16=True, + local_rank=training_conf.local_rank, gradient_checkpointing=training_conf.gradient_checkpointing, gradient_accumulation_steps=training_conf.gradient_accumulation_steps, per_device_train_batch_size=training_conf.per_device_train_batch_size, @@ -192,6 +165,7 @@ if __name__ == "__main__": trainer = SFTTrainer( model, args, + loss_function=training_conf.loss_fn, train_dataset=train, eval_dataset=evals, data_collator=collate_fn, diff --git a/model/supervised_finetuning/utils.py b/model/supervised_finetuning/utils.py index a31f74d3..7147dcb9 100644 --- a/model/supervised_finetuning/utils.py +++ b/model/supervised_finetuning/utils.py @@ -8,14 +8,16 @@ from sklearn.model_selection import train_test_split from torch.utils.data import ConcatDataset, Subset from transformers import AutoModelForCausalLM, AutoTokenizer -SUPPORTED_MODELS = ["galactica", "GPT-JT"] # deprecated .. - def get_tokenizer(conf): tokenizer = AutoTokenizer.from_pretrained(conf.model_name, cache_dir=conf.cache_dir) if "galactica" in conf.model_name: tokenizer.add_special_tokens({"pad_token": "", "eos_token": ""}) + elif "GPT-JT" in conf.model_name: + tokenizer.add_special_tokens({"pad_token": tokenizer.eos_token, "sep_token": "<|extratoken_100|>"}) + elif "codegen" in conf.model_name: + tokenizer.add_special_tokens({"pad_token": "<|endoftext|>", "sep_token": "<|endoftext|>"}) additional_special_tokens = ( [] @@ -30,11 +32,6 @@ def get_tokenizer(conf): def get_model(conf, tokenizer): - if not any([x in conf.model_name for x in SUPPORTED_MODELS]): - raise ValueError( - f"Model {conf.model_name} not supported. Supported models: {SUPPORTED_MODELS}. " - "To include more make sure the masking is dne correctly... (decoder only supported for now)" - ) model = AutoModelForCausalLM.from_pretrained(conf.model_name, cache_dir=conf.cache_dir) From 8e30b419bf1c1c0559fff03770a07ddd5fb668b0 Mon Sep 17 00:00:00 2001 From: theblackcat102 Date: Fri, 6 Jan 2023 16:54:46 +0000 Subject: [PATCH 16/32] [fix] new code complete answer --- .../custom_datasets/prompt_dialogue.py | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/model/supervised_finetuning/custom_datasets/prompt_dialogue.py b/model/supervised_finetuning/custom_datasets/prompt_dialogue.py index 07f32266..08a09113 100644 --- a/model/supervised_finetuning/custom_datasets/prompt_dialogue.py +++ b/model/supervised_finetuning/custom_datasets/prompt_dialogue.py @@ -30,26 +30,16 @@ class PromptGeneratedDataset(Dataset): answer = "" self.pairs = [] with open(chat_dialogue, "r") as f: - - for line in f: - try: - line = line.rstrip() - if len(line) == 0: - continue - elif line == "<|endoftext|>" and len(question) > 0 and len(answer) > 0: + corpus = f.read().split('<|endoftext|>') + for dialogue in corpus: + dialogue = dialogue.strip() + if 'Rosey:' in dialogue: + user, bot = dialogue.split('Rosey:', maxsplit=1) + question = user.split(":", maxsplit=1)[1].strip() + answer = bot.strip() + if len(answer) and len(question): self.pairs.append((question, answer)) - question = "" - answer = "" - elif line[:4] == "User": - question = line.split(":", maxsplit=1)[1] - elif line == "Rosey:": - question = "" - answer = "" - elif len(line) > 4: # should be bot answer - answer = line.split(":", maxsplit=1)[1] - except IndexError: - question = "" - answer = "" + if len(question) > 0 and len(answer) > 0: self.pairs.append((question, answer)) From fc6eab9edc868d2328936a18d19e1b37954cc60a Mon Sep 17 00:00:00 2001 From: theblackcat102 Date: Fri, 6 Jan 2023 17:05:47 +0000 Subject: [PATCH 17/32] [fix] new code complete answer and update readme for clarity --- model/supervised_finetuning/README.md | 32 ++++++++++++++++++- .../custom_datasets/prompt_dialogue.py | 7 ++-- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/model/supervised_finetuning/README.md b/model/supervised_finetuning/README.md index 014afa95..ec337a3d 100644 --- a/model/supervised_finetuning/README.md +++ b/model/supervised_finetuning/README.md @@ -23,7 +23,37 @@ open-asisstant dataset are available it will be added here. ## Model -TBD +Normally you should be able to add new models in configs/config.yml + +``` +your-model-name: + learning_rate: 2e-6 + model_name: + weight_decay: 0.01 + max_length: 812 + warmup_steps: 600 + gradient_checkpointing: false + gradient_accumulation_steps: 5 + per_device_train_batch_size: 4 + per_device_eval_batch_size: 4 +``` + +``` +python trainer.py --configs defaults your-model-name +``` + +However, if the model of your choice doesn't have pad_token, eos_token, sep_token, you have to update utils.py `get_tokenizer` to use the right token. + + +## Deepspeed support + +You can edit the configs/zero_config.json and use any stage you wish. The current config uses zero-stage 3. For more details on how to setup the config checkout [this page](https://www.deepspeed.ai/tutorials/zero/) + +Once you are satisfy with your deepzero config, you can add --deepspeed flag at the end to trigger deepspeed + +``` +python trainer.py --configs defaults your-model-name --deepspeed +``` ## Results diff --git a/model/supervised_finetuning/custom_datasets/prompt_dialogue.py b/model/supervised_finetuning/custom_datasets/prompt_dialogue.py index 08a09113..17911141 100644 --- a/model/supervised_finetuning/custom_datasets/prompt_dialogue.py +++ b/model/supervised_finetuning/custom_datasets/prompt_dialogue.py @@ -30,17 +30,16 @@ class PromptGeneratedDataset(Dataset): answer = "" self.pairs = [] with open(chat_dialogue, "r") as f: - corpus = f.read().split('<|endoftext|>') + corpus = f.read().split("<|endoftext|>") for dialogue in corpus: dialogue = dialogue.strip() - if 'Rosey:' in dialogue: - user, bot = dialogue.split('Rosey:', maxsplit=1) + if "Rosey:" in dialogue: + user, bot = dialogue.split("Rosey:", maxsplit=1) question = user.split(":", maxsplit=1)[1].strip() answer = bot.strip() if len(answer) and len(question): self.pairs.append((question, answer)) - if len(question) > 0 and len(answer) > 0: self.pairs.append((question, answer)) From 577b14a702784654ac4a0019b885639db9aa5d93 Mon Sep 17 00:00:00 2001 From: theblackcat102 Date: Fri, 6 Jan 2023 17:11:06 +0000 Subject: [PATCH 18/32] [fix] push fix by linter --- model/supervised_finetuning/README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/model/supervised_finetuning/README.md b/model/supervised_finetuning/README.md index ec337a3d..bd202397 100644 --- a/model/supervised_finetuning/README.md +++ b/model/supervised_finetuning/README.md @@ -42,14 +42,17 @@ your-model-name: python trainer.py --configs defaults your-model-name ``` -However, if the model of your choice doesn't have pad_token, eos_token, sep_token, you have to update utils.py `get_tokenizer` to use the right token. - +However, if the model of your choice doesn't have pad_token, eos_token, +sep_token, you have to update utils.py `get_tokenizer` to use the right token. ## Deepspeed support -You can edit the configs/zero_config.json and use any stage you wish. The current config uses zero-stage 3. For more details on how to setup the config checkout [this page](https://www.deepspeed.ai/tutorials/zero/) +You can edit the configs/zero_config.json and use any stage you wish. The +current config uses zero-stage 3. For more details on how to setup the config +checkout [this page](https://www.deepspeed.ai/tutorials/zero/) -Once you are satisfy with your deepzero config, you can add --deepspeed flag at the end to trigger deepspeed +Once you are satisfy with your deepzero config, you can add --deepspeed flag at +the end to trigger deepspeed ``` python trainer.py --configs defaults your-model-name --deepspeed From 50e7472ae6bc9156684f6e44fad83fecd7428b52 Mon Sep 17 00:00:00 2001 From: theblackcat102 Date: Fri, 6 Jan 2023 17:14:19 +0000 Subject: [PATCH 19/32] [fix] push fix by linter --- model/supervised_finetuning/trainer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/model/supervised_finetuning/trainer.py b/model/supervised_finetuning/trainer.py index 2e59edcc..a66f2d11 100644 --- a/model/supervised_finetuning/trainer.py +++ b/model/supervised_finetuning/trainer.py @@ -5,7 +5,6 @@ from typing import Any, Dict, List, Optional, Tuple, Union import torch from torch import nn -from torch.utils.data import Dataset from transformers import PreTrainedModel, Trainer, TrainingArguments, get_cosine_schedule_with_warmup from utils import get_dataset, get_loss, get_model, get_tokenizer, read_yamls From ef02693ac9b4705b4ba026a05436709a479dcec7 Mon Sep 17 00:00:00 2001 From: Sotirios Anagnostidis Date: Fri, 6 Jan 2023 18:24:28 +0100 Subject: [PATCH 20/32] quantization --- .../supervised_finetuning/configs/config.yaml | 2 +- .../supervised_finetuning/models/__init__.py | 25 +++- model/supervised_finetuning/models/gptj.py | 60 +++++----- model/supervised_finetuning/trainer.py | 107 ++++++++++++------ model/supervised_finetuning/utils.py | 36 +----- 5 files changed, 128 insertions(+), 102 deletions(-) diff --git a/model/supervised_finetuning/configs/config.yaml b/model/supervised_finetuning/configs/config.yaml index ded4363e..2f529e85 100644 --- a/model/supervised_finetuning/configs/config.yaml +++ b/model/supervised_finetuning/configs/config.yaml @@ -50,4 +50,4 @@ debug: gradient_accumulation_steps: 2 per_device_train_batch_size: 1 per_device_eval_batch_size: 1 - quantization: 8bit \ No newline at end of file + quantization: \ No newline at end of file diff --git a/model/supervised_finetuning/models/__init__.py b/model/supervised_finetuning/models/__init__.py index 673f4085..b99053e3 100644 --- a/model/supervised_finetuning/models/__init__.py +++ b/model/supervised_finetuning/models/__init__.py @@ -1,8 +1,31 @@ from transformers import AutoModelForCausalLM from .gptj import get_model as get_gptj_model +SUPPORTED_MODELS = ["galactica", "gpt-j"] + + +def freeze_top_n_layers(model, target_layers): + # its possible we can simply detect which module is a ModuleList + # and simply freeze the module without doing string parsing + for name, param in model.named_parameters(): + if "embed" in name: + param.requires_grad = False + elif ".layer" in name or ".h." in name: + tokens = name.split(".") + layer_ = None + for token in tokens: + if token.isdigit(): + layer_ = int(token) + break + + if layer_ is not None and layer_ < target_layers: + # print('freeze ', layer_, name) + param.requires_grad = False + return model + + def get_specific_model(model_name, cache_dir, quantization): if "gpt-j" in model_name.lower(): return get_gptj_model(model_name, cache_dir, quantization) else: - return AutoModelForCausalLM.from_pretrained(conf.model_name, cache_dir=conf.cache_dir) \ No newline at end of file + return AutoModelForCausalLM.from_pretrained(model_name, cache_dir=cache_dir) diff --git a/model/supervised_finetuning/models/gptj.py b/model/supervised_finetuning/models/gptj.py index 11234abd..3cbec3ce 100644 --- a/model/supervised_finetuning/models/gptj.py +++ b/model/supervised_finetuning/models/gptj.py @@ -19,38 +19,32 @@ class FrozenBNBLinear(nn.Module): self.register_buffer("code", code.requires_grad_(False)) self.adapter = None self.bias = bias - + def forward(self, input): output = DequantizeAndLinear.apply(input, self.weight, self.absmax, self.code, self.bias) if self.adapter: output += self.adapter(input) return output - + @classmethod def from_linear(cls, linear: nn.Linear) -> "FrozenBNBLinear": weights_int8, state = quantize_blockise_lowmemory(linear.weight) return cls(weights_int8, *state, linear.bias) - + def __repr__(self): return f"{self.__class__.__name__}({self.in_features}, {self.out_features})" - - -class DequantizeAndLinear(torch.autograd.Function): + + +class DequantizeAndLinear(torch.autograd.Function): @staticmethod @custom_fwd - def forward( - ctx, - input: torch.Tensor, - weights_quantized: torch.ByteTensor, - absmax: torch.FloatTensor, - code: torch.FloatTensor, - bias: torch.FloatTensor, - ): + def forward(ctx, input: torch.Tensor, weights_quantized: torch.ByteTensor, + absmax: torch.FloatTensor, code: torch.FloatTensor, bias: torch.FloatTensor): weights_deq = dequantize_blockwise(weights_quantized, absmax=absmax, code=code) ctx.save_for_backward(input, weights_quantized, absmax, code) ctx._has_bias = bias is not None return F.linear(input, weights_deq, bias) - + @staticmethod @custom_bwd def backward(ctx, grad_output: torch.Tensor): @@ -61,8 +55,8 @@ class DequantizeAndLinear(torch.autograd.Function): grad_input = grad_output @ weights_deq grad_bias = grad_output.flatten(0, -2).sum(dim=0) if ctx._has_bias else None return grad_input, None, None, None, grad_bias - - + + class FrozenBNBEmbedding(nn.Module): def __init__(self, weight, absmax, code): super().__init__() @@ -71,7 +65,7 @@ class FrozenBNBEmbedding(nn.Module): self.register_buffer("absmax", absmax.requires_grad_(False)) self.register_buffer("code", code.requires_grad_(False)) self.adapter = None - + def forward(self, input, **kwargs): with torch.no_grad(): # note: both quantuized weights and input indices are *not* differentiable @@ -79,41 +73,41 @@ class FrozenBNBEmbedding(nn.Module): output = F.embedding(input, weight_deq, **kwargs) if self.adapter: output += self.adapter(input) - return output - + return output + @classmethod def from_embedding(cls, embedding: nn.Embedding) -> "FrozenBNBEmbedding": weights_int8, state = quantize_blockise_lowmemory(embedding.weight) return cls(weights_int8, *state) - + def __repr__(self): return f"{self.__class__.__name__}({self.num_embeddings}, {self.embedding_dim})" - - -def quantize_blockise_lowmemory(matrix: torch.Tensor, chunk_size: int = 2**20): + + +def quantize_blockise_lowmemory(matrix: torch.Tensor, chunk_size: int = 2 ** 20): assert chunk_size % 4096 == 0 code = None chunks = [] absmaxes = [] flat_tensor = matrix.view(-1) for i in range((matrix.numel() - 1) // chunk_size + 1): - input_chunk = flat_tensor[i * chunk_size : (i + 1) * chunk_size].clone() + input_chunk = flat_tensor[i * chunk_size: (i + 1) * chunk_size].clone() quantized_chunk, (absmax_chunk, code) = quantize_blockwise(input_chunk, code=code) chunks.append(quantized_chunk) absmaxes.append(absmax_chunk) - + matrix_i8 = torch.cat(chunks).reshape_as(matrix) absmax = torch.cat(absmaxes) return matrix_i8, (absmax, code) - - + + def convert_to_int8(model): """Convert linear and embedding modules to 8-bit with optional adapters""" for module in list(model.modules()): for name, child in module.named_children(): if isinstance(child, nn.Linear): print(name, child) - setattr( + setattr( module, name, FrozenBNBLinear( @@ -131,7 +125,7 @@ def convert_to_int8(model): weight=torch.zeros(child.num_embeddings, child.embedding_dim, dtype=torch.uint8), absmax=torch.zeros((child.weight.numel() - 1) // 4096 + 1), code=torch.zeros(256), - ), + ) ) @@ -147,13 +141,14 @@ class GPTJModel(transformers.models.gptj.modeling_gptj.GPTJModel): def __init__(self, config): super().__init__(config) convert_to_int8(self) - + class GPTJForCausalLM(transformers.models.gptj.modeling_gptj.GPTJForCausalLM): def __init__(self, config): super().__init__(config) convert_to_int8(self) + def add_adapters(model, adapter_dim=16): assert adapter_dim > 0 @@ -176,9 +171,10 @@ def get_model(model_name, cache_dir, quantization): if quantization is None: model = AutoModelForCausalLM.from_pretrained(model_name, cache_dir=cache_dir) elif quantization == "8bit": + print("Loading 8-bit model") transformers.models.gptj.modeling_gptj.GPTJBlock = GPTJBlock model = AutoModelForCausalLM.from_pretrained(model_name, cache_dir=cache_dir) - add_adapters(gpt) + add_adapters(model) else: raise ValueError(f"Unknown quantization {quantization}") diff --git a/model/supervised_finetuning/trainer.py b/model/supervised_finetuning/trainer.py index 0db0f443..590f9bbd 100644 --- a/model/supervised_finetuning/trainer.py +++ b/model/supervised_finetuning/trainer.py @@ -74,6 +74,7 @@ class SFTTrainer(Trainer): self.loss_fct = get_loss(args.loss_function) def create_optimizer_and_scheduler(self, num_training_steps: int): + print("Optimizer") if self.args.quantization == "8bit": self.optimizer = bnb.optim.Adam8bit(model.parameters(), lr=0.001, betas=(0.9, 0.995)) else: @@ -174,40 +175,76 @@ if __name__ == "__main__": tokenizer = get_tokenizer(training_conf) model = get_model(training_conf, tokenizer) - train, evals, collate_fn = get_dataset(training_conf, tokenizer) - assert len(evals) > 0 + ### + from datasets import load_dataset + from bitsandbytes.optim import Adam8bit + from torch.nn import functional as F + from tqdm import tqdm - args = CustomTrainingArguments( - output_dir=f"{training_conf.model_name}-{training_conf.log_dir}-finetuned", - num_train_epochs=training_conf.num_train_epochs, - warmup_steps=training_conf.warmup_steps, - loss_function=training_conf.loss_fn, - learning_rate=float(training_conf.learning_rate), - fp16=True, - gradient_checkpointing=training_conf.gradient_checkpointing, - gradient_accumulation_steps=training_conf.gradient_accumulation_steps, - per_device_train_batch_size=training_conf.per_device_train_batch_size, - per_device_eval_batch_size=training_conf.per_device_eval_batch_size, - weight_decay=training_conf.weight_decay, - max_grad_norm=training_conf.max_grad_norm, - logging_steps=training_conf.logging_steps, - save_total_limit=training_conf.save_total_limit, - evaluation_strategy="steps", - eval_steps=training_conf.eval_steps, - save_steps=training_conf.save_steps, - eval_accumulation_steps=training_conf.eval_accumulation_steps, - report_to="wandb", - quantization=training_conf.quantization, - ) + gpt = model.to("cuda") - trainer = SFTTrainer( - model, - args, - train_dataset=train, - eval_dataset=evals, - data_collator=collate_fn, - tokenizer=tokenizer, - compute_metrics=compute_metrics, - preprocess_logits_for_metrics=preprocess_logits_for_metrics, - ) - trainer.train() + gpt.gradient_checkpointing_enable() + + codeparrot = load_dataset("transformersbook/codeparrot-train", streaming=True, cache_dir=training_conf.cache_dir) + optimizer = Adam8bit(gpt.parameters(), lr=1e-5) + + with torch.cuda.amp.autocast(): + for row in tqdm(codeparrot["train"]): + if len(row["content"]) <= 1: + continue + + batch = tokenizer(row["content"], truncation=True, max_length=128, return_tensors="pt") + batch = {k: v.cuda() for k, v in batch.items()} + + out = gpt.forward( + **batch, + ) + + loss = F.cross_entropy( + out.logits[:, :-1, :].flatten(0, -2), batch["input_ids"][:, 1:].flatten(), reduction="mean" + ) + print(loss) + loss.backward() + + optimizer.step() + optimizer.zero_grad() + ### + + + # train, evals, collate_fn = get_dataset(training_conf, tokenizer) + # assert len(evals) > 0 + + # args = CustomTrainingArguments( + # output_dir=f"{training_conf.model_name}-{training_conf.log_dir}-finetuned", + # num_train_epochs=training_conf.num_train_epochs, + # warmup_steps=training_conf.warmup_steps, + # loss_function=training_conf.loss_fn, + # learning_rate=float(training_conf.learning_rate), + # fp16=True, + # gradient_checkpointing=training_conf.gradient_checkpointing, + # gradient_accumulation_steps=training_conf.gradient_accumulation_steps, + # per_device_train_batch_size=training_conf.per_device_train_batch_size, + # per_device_eval_batch_size=training_conf.per_device_eval_batch_size, + # weight_decay=training_conf.weight_decay, + # max_grad_norm=training_conf.max_grad_norm, + # logging_steps=training_conf.logging_steps, + # save_total_limit=training_conf.save_total_limit, + # evaluation_strategy="steps", + # eval_steps=training_conf.eval_steps, + # save_steps=training_conf.save_steps, + # eval_accumulation_steps=training_conf.eval_accumulation_steps, + # report_to="wandb", + # quantization=training_conf.quantization, + # ) + + # trainer = SFTTrainer( + # model, + # args, + # train_dataset=train, + # eval_dataset=evals, + # data_collator=collate_fn, + # tokenizer=tokenizer, + # compute_metrics=compute_metrics, + # preprocess_logits_for_metrics=preprocess_logits_for_metrics, + # ) + # trainer.train() diff --git a/model/supervised_finetuning/utils.py b/model/supervised_finetuning/utils.py index 318aeb5e..c3cc512c 100644 --- a/model/supervised_finetuning/utils.py +++ b/model/supervised_finetuning/utils.py @@ -7,9 +7,7 @@ from losses import CrossEntropyLoss from sklearn.model_selection import train_test_split from torch.utils.data import ConcatDataset, Subset from transformers import AutoTokenizer -from models import get_specific_model - -SUPPORTED_MODELS = ["galactica", "GPT-JT"] # deprecated .. +from models import get_specific_model, SUPPORTED_MODELS, freeze_top_n_layers def get_tokenizer(conf): @@ -31,10 +29,10 @@ def get_tokenizer(conf): def get_model(conf, tokenizer): - if not any([x in conf.model_name for x in SUPPORTED_MODELS]): + if not any([x in conf.model_name.lower() for x in SUPPORTED_MODELS]): raise ValueError( f"Model {conf.model_name} not supported. Supported models: {SUPPORTED_MODELS}. " - "To include more make sure the masking is dne correctly... (decoder only supported for now)" + "To include more make sure the masking is done correctly... (decoder only supported for now)" ) model = get_specific_model(conf.model_name, conf.cache_dir, conf.quantization) @@ -96,31 +94,3 @@ def train_val_dataset(dataset, val_split=0.2): list(range(len(dataset))), test_size=val_split, random_state=666, shuffle=True ) return Subset(dataset, train_idx), Subset(dataset, val_idx) - - -def freeze_top_n_layers(model, target_layers): - # its possible we can simply detect which module is a ModuleList - # and simply freeze the module without doing string parsing - for name, param in model.named_parameters(): - if "embed" in name: - param.requires_grad = False - elif ".layer" in name or ".h." in name: - tokens = name.split(".") - layer_ = None - for token in tokens: - if token.isdigit(): - layer_ = int(token) - break - - if layer_ is not None and layer_ < target_layers: - # print('freeze ', layer_, name) - param.requires_grad = False - return model - - -if __name__ == "__main__": - from transformers import AutoModelForSequenceClassification - - model = AutoModelForSequenceClassification.from_pretrained("bigscience/bloomz-560m") - freeze_top_n_layers(model, 10) - print(model.state_dict().keys()) From 91853753a8759208ac9c9c8cdb07ef3ae2fffdba Mon Sep 17 00:00:00 2001 From: Sotirios Anagnostidis Date: Fri, 6 Jan 2023 18:25:20 +0100 Subject: [PATCH 21/32] conf --- model/supervised_finetuning/configs/config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/model/supervised_finetuning/configs/config.yaml b/model/supervised_finetuning/configs/config.yaml index 2f529e85..64c024c2 100644 --- a/model/supervised_finetuning/configs/config.yaml +++ b/model/supervised_finetuning/configs/config.yaml @@ -47,7 +47,8 @@ gpt-jt: debug: eval_steps: 20 eval_size: 100 + model_name: EleutherAI/gpt-j-6B gradient_accumulation_steps: 2 per_device_train_batch_size: 1 per_device_eval_batch_size: 1 - quantization: \ No newline at end of file + quantization: 8bit \ No newline at end of file From 69bc799cd9bdb80bcaa8388368b75551d573eac0 Mon Sep 17 00:00:00 2001 From: Oliver Stanley Date: Fri, 6 Jan 2023 17:39:04 +0000 Subject: [PATCH 22/32] 344: Create tasks for text labels (#381) * Implement label task for initial prompts and replies * Resolve formatting * Include missing argument * Modify text_labels API to match new model, update DB schema accordingly * Send valid labels as part of label tasks * Send correctly formatted valid_labels list * Fix request format * Fix request details for text-frontend reply label task * Include message_id in tasks * Address review comments * Fix alembic tree --- ...5-20cd871f4ec7_added_user_to_textlabels.py | 30 ++++++++++ backend/oasst_backend/api/v1/tasks.py | 39 +++++++++++++ backend/oasst_backend/api/v1/text_labels.py | 14 ++--- backend/oasst_backend/models/db_payload.py | 35 +++++++++++ backend/oasst_backend/models/text_labels.py | 1 + backend/oasst_backend/prompt_repository.py | 35 +++++++++-- discord-bot/bot/extensions/work.py | 52 +++++++++++++++++ discord-bot/bot/messages.py | 58 +++++++++++++++++++ oasst-shared/oasst_shared/api_client.py | 6 ++ oasst-shared/oasst_shared/schemas/protocol.py | 58 ++++++++++++++++--- text-frontend/__main__.py | 54 +++++++++++++++++ 11 files changed, 357 insertions(+), 25 deletions(-) create mode 100644 backend/alembic/versions/2023_01_05_1745-20cd871f4ec7_added_user_to_textlabels.py diff --git a/backend/alembic/versions/2023_01_05_1745-20cd871f4ec7_added_user_to_textlabels.py b/backend/alembic/versions/2023_01_05_1745-20cd871f4ec7_added_user_to_textlabels.py new file mode 100644 index 00000000..f042642e --- /dev/null +++ b/backend/alembic/versions/2023_01_05_1745-20cd871f4ec7_added_user_to_textlabels.py @@ -0,0 +1,30 @@ +"""Added user to TextLabels + +Revision ID: 20cd871f4ec7 +Revises: d4161e384f83 +Create Date: 2023-01-05 17:45:15.696468 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "20cd871f4ec7" +down_revision = "3b0adfadbef9" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("text_labels", sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False)) + op.create_foreign_key(None, "text_labels", "user", ["user_id"], ["id"]) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, "text_labels", type_="foreignkey") + op.drop_column("text_labels", "user_id") + # ### end Alembic commands ### diff --git a/backend/oasst_backend/api/v1/tasks.py b/backend/oasst_backend/api/v1/tasks.py index 05dc92a9..9f81eabb 100644 --- a/backend/oasst_backend/api/v1/tasks.py +++ b/backend/oasst_backend/api/v1/tasks.py @@ -119,6 +119,38 @@ def generate_task( conversation=protocol_schema.Conversation(messages=task_messages), replies=replies, ) + + case protocol_schema.TaskRequestType.label_initial_prompt: + logger.info("Generating a LabelInitialPromptTask.") + message = pr.fetch_random_initial_prompts(1)[0] + task = protocol_schema.LabelInitialPromptTask( + message_id=message.id, + prompt=message.payload.payload.text, + valid_labels=list(map(lambda x: x.value, protocol_schema.TextLabel)), + ) + + case protocol_schema.TaskRequestType.label_prompter_reply: + logger.info("Generating a LabelPrompterReplyTask.") + conversation, messages = pr.fetch_multiple_random_replies(max_size=1, message_role="assistant") + message = messages[0].payload.payload.text + task = protocol_schema.LabelPrompterReplyTask( + message_id=message.id, + conversation=conversation, + reply=message, + valid_labels=list(map(lambda x: x.value, protocol_schema.TextLabel)), + ) + + case protocol_schema.TaskRequestType.label_assistant_reply: + logger.info("Generating a LabelAssistantReplyTask.") + conversation, messages = pr.fetch_multiple_random_replies(max_size=1, message_role="prompter") + message = messages[0].payload.payload.text + task = protocol_schema.LabelAssistantReplyTask( + message_id=message.id, + conversation=conversation, + reply=message, + valid_labels=list(map(lambda x: x.value, protocol_schema.TextLabel)), + ) + case _: raise OasstError("Invalid request type", OasstErrorCode.TASK_INVALID_REQUEST_TYPE) @@ -256,6 +288,13 @@ def tasks_interaction( pr.store_ranking(interaction) # here we would store the ranking in the database return protocol_schema.TaskDone() + case protocol_schema.TextLabels: + logger.info( + f"Frontend reports labels of {interaction.message_id=} with {interaction.labels=} by {interaction.user=}." + ) + # TODO: check if the labels are valid? + pr.store_text_labels(interaction) + return protocol_schema.TaskDone() case _: raise OasstError("Invalid response type.", OasstErrorCode.TASK_INVALID_RESPONSE_TYPE) except OasstError: diff --git a/backend/oasst_backend/api/v1/text_labels.py b/backend/oasst_backend/api/v1/text_labels.py index 0613711c..97422119 100644 --- a/backend/oasst_backend/api/v1/text_labels.py +++ b/backend/oasst_backend/api/v1/text_labels.py @@ -1,4 +1,3 @@ -import pydantic from fastapi import APIRouter, Depends, HTTPException from fastapi.security.api_key import APIKey from loguru import logger @@ -11,17 +10,12 @@ from starlette.status import HTTP_204_NO_CONTENT, HTTP_400_BAD_REQUEST router = APIRouter() -class LabelTextRequest(pydantic.BaseModel): - text_labels: protocol_schema.TextLabels - user: protocol_schema.User - - @router.post("/", status_code=HTTP_204_NO_CONTENT) def label_text( *, db: Session = Depends(deps.get_db), api_key: APIKey = Depends(deps.get_api_key), - request: LabelTextRequest, + text_labels: protocol_schema.TextLabels, ) -> None: """ Label a piece of text. @@ -29,9 +23,9 @@ def label_text( api_client = deps.api_auth(api_key, db) try: - logger.info(f"Labeling text {request=}.") - pr = PromptRepository(db, api_client, user=request.user) - pr.store_text_labels(request.text_labels) + logger.info(f"Labeling text {text_labels=}.") + pr = PromptRepository(db, api_client, user=text_labels.user) + pr.store_text_labels(text_labels) except Exception: logger.exception("Failed to store label.") diff --git a/backend/oasst_backend/models/db_payload.py b/backend/oasst_backend/models/db_payload.py index 9a6fabb6..fed60dd8 100644 --- a/backend/oasst_backend/models/db_payload.py +++ b/backend/oasst_backend/models/db_payload.py @@ -1,4 +1,5 @@ from typing import Literal +from uuid import UUID from oasst_backend.models.payload_column_type import payload_type from oasst_shared.schemas import protocol as protocol_schema @@ -91,3 +92,37 @@ class RankAssistantRepliesPayload(RankConversationRepliesPayload): """A task to rank a set of assistant replies to a conversation.""" type: Literal["rank_assistant_replies"] = "rank_assistant_replies" + + +@payload_type +class LabelInitialPromptPayload(TaskPayload): + """A task to label an initial prompt.""" + + type: Literal["label_initial_prompt"] = "label_initial_prompt" + message_id: UUID + prompt: str + valid_labels: list[str] + + +@payload_type +class LabelConversationReplyPayload(TaskPayload): + """A task to label a conversation reply.""" + + message_id: UUID + conversation: protocol_schema.Conversation + reply: str + valid_labels: list[str] + + +@payload_type +class LabelPrompterReplyPayload(LabelConversationReplyPayload): + """A task to label a prompter reply.""" + + type: Literal["label_prompter_reply"] = "label_prompter_reply" + + +@payload_type +class LabelAssistantReplyPayload(LabelConversationReplyPayload): + """A task to label an assistant reply.""" + + type: Literal["label_assistant_reply"] = "label_assistant_reply" diff --git a/backend/oasst_backend/models/text_labels.py b/backend/oasst_backend/models/text_labels.py index ec10dca6..e6878a87 100644 --- a/backend/oasst_backend/models/text_labels.py +++ b/backend/oasst_backend/models/text_labels.py @@ -15,6 +15,7 @@ class TextLabels(SQLModel, table=True): pg.UUID(as_uuid=True), primary_key=True, default=uuid4, server_default=sa.text("gen_random_uuid()") ), ) + user_id: UUID = Field(sa_column=sa.Column(pg.UUID(as_uuid=True), sa.ForeignKey("user.id"), nullable=False)) created_date: Optional[datetime] = Field( sa_column=sa.Column(sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()), ) diff --git a/backend/oasst_backend/prompt_repository.py b/backend/oasst_backend/prompt_repository.py index 157e42a7..7c7dd7b6 100644 --- a/backend/oasst_backend/prompt_repository.py +++ b/backend/oasst_backend/prompt_repository.py @@ -282,16 +282,39 @@ class PromptRepository: payload = db_payload.AssistantReplyPayload(type=task.type, conversation=task.conversation) case protocol_schema.RankInitialPromptsTask: - payload = db_payload.RankInitialPromptsPayload(tpye=task.type, prompts=task.prompts) + payload = db_payload.RankInitialPromptsPayload(type=task.type, prompts=task.prompts) case protocol_schema.RankPrompterRepliesTask: payload = db_payload.RankPrompterRepliesPayload( - tpye=task.type, conversation=task.conversation, replies=task.replies + type=task.type, conversation=task.conversation, replies=task.replies ) case protocol_schema.RankAssistantRepliesTask: payload = db_payload.RankAssistantRepliesPayload( - tpye=task.type, conversation=task.conversation, replies=task.replies + type=task.type, conversation=task.conversation, replies=task.replies + ) + + case protocol_schema.LabelInitialPromptTask: + payload = db_payload.LabelInitialPromptPayload( + type=task.type, message_id=task.message_id, prompt=task.prompt, valid_labels=task.valid_labels + ) + + case protocol_schema.LabelPrompterReplyTask: + payload = db_payload.LabelPrompterReplyPayload( + type=task.type, + message_id=task.message_id, + conversation=task.conversation, + reply=task.reply, + valid_labels=task.valid_labels, + ) + + case protocol_schema.LabelAssistantReplyTask: + payload = db_payload.LabelAssistantReplyPayload( + type=task.type, + message_id=task.message_id, + conversation=task.conversation, + reply=task.reply, + valid_labels=task.valid_labels, ) case _: @@ -388,12 +411,12 @@ class PromptRepository: def store_text_labels(self, text_labels: protocol_schema.TextLabels) -> TextLabels: model = TextLabels( api_client_id=self.api_client.id, + message_id=text_labels.message_id, + user_id=self.user_id, text=text_labels.text, labels=text_labels.labels, ) - if text_labels.has_message_id: - self.fetch_message_by_frontend_message_id(text_labels.message_id, fail_if_missing=True) - model.message_id = text_labels.message_id + self.db.add(model) self.db.commit() self.db.refresh(model) diff --git a/discord-bot/bot/extensions/work.py b/discord-bot/bot/extensions/work.py index 0561039d..51daca3b 100644 --- a/discord-bot/bot/extensions/work.py +++ b/discord-bot/bot/extensions/work.py @@ -10,10 +10,14 @@ import miru from aiosqlite import Connection from bot.messages import ( assistant_reply_message, + confirm_label_response_message, confirm_ranking_response_message, confirm_text_response_message, initial_prompt_message, invalid_user_input_embed, + label_assistant_reply_message, + label_initial_prompt_message, + label_prompter_reply_message, plain_embed, prompter_reply_message, rank_assistant_reply_message, @@ -145,6 +149,8 @@ async def _handle_task(ctx: lightbulb.Context, task_type: TaskRequestType) -> No content = confirm_ranking_response_message(event.content, task.replies) elif isinstance(task, protocol_schema.RankInitialPromptsTask): content = confirm_ranking_response_message(event.content, task.prompts) + elif isinstance(task, protocol_schema.LabelConversationReplyTask | protocol_schema.LabelInitialPromptTask): + content = confirm_label_response_message(event.content) elif isinstance(task, protocol_schema.ReplyToConversationTask | protocol_schema.InitialPromptTask): content = confirm_text_response_message(event.content) else: @@ -171,6 +177,17 @@ async def _handle_task(ctx: lightbulb.Context, task_type: TaskRequestType) -> No auth_method="discord", id=str(ctx.author.id), display_name=ctx.author.username ), ) + elif isinstance(task, protocol_schema.LabelConversationReplyTask | protocol_schema.LabelInitialPromptTask): + labels = event.content.replace(" ", "").split(",") + labels_dict = {label: 1 if label in labels else 0 for label in task.valid_labels} + + reply = protocol_schema.TextLabels( + message_id=task.message_id, + labels=labels_dict, + user=protocol_schema.User( + auth_method="discord", id=str(ctx.author.id), display_name=ctx.author.username + ), + ) elif isinstance(task, protocol_schema.ReplyToConversationTask | protocol_schema.InitialPromptTask): reply = protocol_schema.TextReplyToMessage( message_id=str(msg_id), @@ -300,6 +317,21 @@ async def _send_task( logger.debug("sending rank assistant reply task") content = rank_assistant_reply_message(task) + elif task.type == TaskRequestType.label_initial_prompt: + assert isinstance(task, protocol_schema.LabelInitialPromptTask) + logger.debug("sending label initial prompt task") + content = label_initial_prompt_message(task) + + elif task.type == TaskRequestType.label_prompter_reply: + assert isinstance(task, protocol_schema.LabelPrompterReplyTask) + logger.debug("sending label prompter reply task") + content = label_prompter_reply_message(task) + + elif task.type == TaskRequestType.label_assistant_reply: + assert isinstance(task, protocol_schema.LabelAssistantReplyTask) + logger.debug("sending label assistant reply task") + content = label_assistant_reply_message(task) + elif task.type == TaskRequestType.prompter_reply: assert isinstance(task, protocol_schema.PrompterReplyTask) logger.debug("sending user reply task") @@ -382,6 +414,26 @@ def _validate_user_input(content: str | None, task: protocol_schema.Task) -> tup "Message must contain numbers for all prompts.", ) + # Labels tasks + elif task.type in ( + TaskRequestType.label_initial_prompt, + TaskRequestType.label_prompter_reply, + TaskRequestType.label_assistant_reply, + ): + assert isinstance( + task, + protocol_schema.LabelInitialPromptTask + | protocol_schema.LabelPrompterReplyTask + | protocol_schema.LabelAssistantReplyTask, + ) + + labels = content.replace(" ", "").split(",") + valid_labels = set(task.valid_labels) + return ( + set(labels).issubset(valid_labels), + "Message must only contain labels from predefined set of labels.", + ) + elif task.type == TaskRequestType.summarize_story: raise NotImplementedError elif task.type == TaskRequestType.rate_summary: diff --git a/discord-bot/bot/messages.py b/discord-bot/bot/messages.py index 8db54e37..c1a6d355 100644 --- a/discord-bot/bot/messages.py +++ b/discord-bot/bot/messages.py @@ -33,6 +33,10 @@ def _ranking_prompt(text: str) -> str: return f":trophy: _{text}_" +def _label_prompt(text: str) -> str: + return f":question: _{text}" + + def _response_prompt(text: str) -> str: return f":speech_balloon: _{text}_" @@ -129,6 +133,49 @@ def rank_assistant_reply_message(task: protocol_schema.RankAssistantRepliesTask) """ +def label_initial_prompt_message(task: protocol_schema.LabelInitialPromptTask) -> str: + """Creates the message that gets sent to users when they request a `label_initial_prompt` task.""" + return f"""\ + +{_h1("LABEL INITIAL PROMPT")} + + +{task.prompt} + +{_label_prompt("Reply with labels for the prompt separated by commas (example: 'profanity,misleading')")} +""" + + +def label_prompter_reply_message(task: protocol_schema.LabelPrompterReplyTask) -> str: + """Creates the message that gets sent to users when they request a `label_prompter_reply` task.""" + return f"""\ + +{_h1("LABEL PROMPTER REPLY")} + + +{_conversation(task.conversation)} +{_user(None)} +{task.reply} + +{_label_prompt("Reply with labels for the reply separated by commas (example: 'profanity,misleading')")} +""" + + +def label_assistant_reply_message(task: protocol_schema.LabelAssistantReplyTask) -> str: + """Creates the message that gets sent to users when they request a `label_assistant_reply` task.""" + return f"""\ + +{_h1("LABEL ASSISTANT REPLY")} + + +{_conversation(task.conversation)} +{_assistant(None)} +{task.reply} + +{_label_prompt("Reply with labels for the reply separated by commas (example: 'profanity,misleading')")} +""" + + def prompter_reply_message(task: protocol_schema.PrompterReplyTask) -> str: """Creates the message that gets sent to users when they request a `prompter_reply` task.""" return f"""\ @@ -175,6 +222,17 @@ def confirm_ranking_response_message(content: str, items: list[str]) -> str: """ +def confirm_label_response_message(content: str) -> str: + user_labels = content.lower().replace(" ", "").split(",") + user_labels_str = ", ".join(user_labels) + + return f"""\ +{_h2("CONFIRM RESPONSE")} + +{user_labels_str} +""" + + ### # Embeds ### diff --git a/oasst-shared/oasst_shared/api_client.py b/oasst-shared/oasst_shared/api_client.py index 404521db..1ee2865b 100644 --- a/oasst-shared/oasst_shared/api_client.py +++ b/oasst-shared/oasst_shared/api_client.py @@ -24,6 +24,9 @@ class TaskType(str, enum.Enum): rank_initial_prompts = "rank_initial_prompts" rank_prompter_replies = "rank_prompter_replies" rank_assistant_replies = "rank_assistant_replies" + label_initial_prompt = "label_initial_prompt" + label_assistant_reply = "label_assistant_reply" + label_prompter_reply = "label_prompter_reply" done = "task_done" @@ -56,6 +59,9 @@ class OasstApiClient: TaskType.rank_initial_prompts: protocol_schema.RankInitialPromptsTask, TaskType.rank_prompter_replies: protocol_schema.RankPrompterRepliesTask, TaskType.rank_assistant_replies: protocol_schema.RankAssistantRepliesTask, + TaskType.label_initial_prompt: protocol_schema.LabelInitialPromptTask, + TaskType.label_prompter_reply: protocol_schema.LabelPrompterReplyTask, + TaskType.label_assistant_reply: protocol_schema.LabelAssistantReplyTask, TaskType.done: protocol_schema.TaskDone, } diff --git a/oasst-shared/oasst_shared/schemas/protocol.py b/oasst-shared/oasst_shared/schemas/protocol.py index e035d387..1cafc93d 100644 --- a/oasst-shared/oasst_shared/schemas/protocol.py +++ b/oasst-shared/oasst_shared/schemas/protocol.py @@ -18,6 +18,9 @@ class TaskRequestType(str, enum.Enum): rank_initial_prompts = "rank_initial_prompts" rank_prompter_replies = "rank_prompter_replies" rank_assistant_replies = "rank_assistant_replies" + label_initial_prompt = "label_initial_prompt" + label_assistant_reply = "label_assistant_reply" + label_prompter_reply = "label_prompter_reply" class User(BaseModel): @@ -169,6 +172,37 @@ class RankAssistantRepliesTask(RankConversationRepliesTask): type: Literal["rank_assistant_replies"] = "rank_assistant_replies" +class LabelInitialPromptTask(Task): + """A task to label an initial prompt.""" + + type: Literal["label_initial_prompt"] = "label_initial_prompt" + message_id: UUID + prompt: str + valid_labels: list[str] + + +class LabelConversationReplyTask(Task): + """A task to label a reply to a conversation.""" + + type: Literal["label_conversation_reply"] = "label_conversation_reply" + conversation: Conversation # the conversation so far + message_id: UUID + reply: str + valid_labels: list[str] + + +class LabelPrompterReplyTask(LabelConversationReplyTask): + """A task to label a prompter reply to a conversation.""" + + type: Literal["label_prompter_reply"] = "label_prompter_reply" + + +class LabelAssistantReplyTask(LabelConversationReplyTask): + """A task to label an assistant reply to a conversation.""" + + type: Literal["label_assistant_reply"] = "label_assistant_reply" + + class TaskDone(Task): """Signals to the frontend that the task is done.""" @@ -187,6 +221,10 @@ AnyTask = Union[ RankConversationRepliesTask, RankPrompterRepliesTask, RankAssistantRepliesTask, + LabelInitialPromptTask, + LabelConversationReplyTask, + LabelPrompterReplyTask, + LabelAssistantReplyTask, ] @@ -222,13 +260,6 @@ class MessageRanking(Interaction): ranking: conlist(item_type=int, min_items=1) -AnyInteraction = Union[ - TextReplyToMessage, - MessageRating, - MessageRanking, -] - - class TextLabel(str, enum.Enum): """A label for a piece of text.""" @@ -256,12 +287,13 @@ class TextLabel(str, enum.Enum): slang = "slang" -class TextLabels(BaseModel): +class TextLabels(Interaction): """A set of labels for a piece of text.""" + type: Literal["text_labels"] = "text_labels" text: str labels: dict[TextLabel, float] - message_id: str | None = None + message_id: UUID @property def has_message_id(self) -> bool: @@ -277,6 +309,14 @@ class TextLabels(BaseModel): return v +AnyInteraction = Union[ + TextReplyToMessage, + MessageRating, + MessageRanking, + TextLabels, +] + + class SystemStats(BaseModel): all: int = 0 active: int = 0 diff --git a/text-frontend/__main__.py b/text-frontend/__main__.py index 39cc7b26..de65749a 100644 --- a/text-frontend/__main__.py +++ b/text-frontend/__main__.py @@ -203,6 +203,60 @@ def main(backend_url: str = "http://127.0.0.1:8080", api_key: str = "DUMMY_KEY") ) tasks.append(new_task) + case "label_initial_prompt": + typer.echo("Label the following prompt:") + typer.echo(task["prompt"]) + # acknowledge task + message_id = _random_message_id() + _post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id}) + + valid_labels = task["valid_labels"] + labels_str: str = typer.prompt("Enter labels, separated by commas") + labels = labels_str.lower().replace(" ", "").split(",") + labels_dict = {label: "1" if label in labels else "0" for label in valid_labels} + + # send ranking + new_task = _post( + "/api/v1/tasks/interaction", + { + "type": "text_labels", + "message_id": task["message_id"], + "text": task["prompt"], + "labels": labels_dict, + "user": USER, + }, + ) + tasks.append(new_task) + + case "label_prompter_reply" | "label_assistant_reply": + typer.echo("Here is the conversation so far:") + for message in task["conversation"]["messages"]: + typer.echo(_render_message(message)) + + typer.echo("Label the following reply:") + typer.echo(task["reply"]) + # acknowledge task + message_id = _random_message_id() + _post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id}) + + valid_labels = task["valid_labels"] + labels_str: str = typer.prompt("Enter labels, separated by commas") + labels = labels_str.lower().replace(" ", "").split(",") + labels_dict = {label: "1" if label in labels else "0" for label in valid_labels} + + # send ranking + new_task = _post( + "/api/v1/tasks/interaction", + { + "type": "text_labels", + "message_id": task["message_id"], + "text": task["prompt"], + "labels": labels_dict, + "user": USER, + }, + ) + tasks.append(new_task) + case "task_done": typer.echo("Task done!") case _: From 8970195c39965ae97ea2a2f7bb48325a2c72da76 Mon Sep 17 00:00:00 2001 From: Alex Ott <66271487+AlexanderHOtt@users.noreply.github.com> Date: Fri, 6 Jan 2023 10:42:56 -0800 Subject: [PATCH 23/32] Add `/help` and `/tutorial` commands and update bot activity (#399) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add help command and update bot activity * remove useless lines * fix help message * rename variable * start tutorial command * Update messages.py Co-authored-by: Andreas Köpf --- discord-bot/bot/__main__.py | 10 +++++- discord-bot/bot/bot.py | 3 ++ discord-bot/bot/extensions/help.py | 43 ++++++++++++++++++++++ discord-bot/bot/messages.py | 58 ++++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 discord-bot/bot/extensions/help.py diff --git a/discord-bot/bot/__main__.py b/discord-bot/bot/__main__.py index 45820f7d..73f8e6b1 100644 --- a/discord-bot/bot/__main__.py +++ b/discord-bot/bot/__main__.py @@ -3,6 +3,7 @@ import logging import os from bot.bot import bot +from hikari.presences import Activity, ActivityType, Status logger = logging.getLogger(__name__) @@ -13,4 +14,11 @@ if __name__ == "__main__": uvloop.install() logger.info("Starting bot") - bot.run() + bot.run( + check_for_updates=True, + activity=Activity( + name="/help", + type=ActivityType.PLAYING, + ), + status=Status.ONLINE, + ) diff --git a/discord-bot/bot/bot.py b/discord-bot/bot/bot.py index 8c604e1a..870f445a 100644 --- a/discord-bot/bot/bot.py +++ b/discord-bot/bot/bot.py @@ -19,6 +19,7 @@ bot = lightbulb.BotApp( default_enabled_guilds=settings.declare_global_commands, owner_ids=settings.owner_ids, intents=hikari.Intents.ALL, + help_class=None, ) @@ -28,10 +29,12 @@ async def on_starting(event: hikari.StartingEvent): miru.install(bot) # component handler bot.load_extensions_from("./bot/extensions") # load extensions + # Database setup bot.d.db = await aiosqlite.connect("./bot/db/database.db") await bot.d.db.executescript(open("./bot/db/schema.sql").read()) await bot.d.db.commit() + # OASST API setup bot.d.oasst_api = OasstApiClient(settings.oasst_api_url, settings.oasst_api_key) # A `dict[hikari.Message | None, UUID | None]]` that maps user IDs to (task msg ID, task UUIDs). diff --git a/discord-bot/bot/extensions/help.py b/discord-bot/bot/extensions/help.py new file mode 100644 index 00000000..b7da3868 --- /dev/null +++ b/discord-bot/bot/extensions/help.py @@ -0,0 +1,43 @@ +"""Custom help command.""" +import lightbulb +from bot.messages import help_message, tutorial_message +from bot.settings import Settings +from hikari.permissions import Permissions +from lightbulb.utils import permissions_for + +plugin = lightbulb.Plugin("HelpPlugin") + +settings = Settings() + + +@plugin.command +@lightbulb.command("help", "Help for the bot.", ephemeral=True) +@lightbulb.implements(lightbulb.SlashCommand, lightbulb.PrefixCommand) +async def help_command(ctx: lightbulb.Context) -> None: + """Help for the bot.""" + can_manage_guild = False + if ctx.guild_id: + member = ctx.bot.cache.get_member(ctx.guild_id, ctx.author.id) or await ctx.bot.rest.fetch_member( + ctx.guild_id, ctx.author.id + ) + can_manage_guild = bool(permissions_for(member) & Permissions.MANAGE_GUILD) + + await ctx.respond(help_message(can_manage_guild, ctx.author.id in settings.owner_ids)) + + +@plugin.command +@lightbulb.command("tutorial", "A tutorial for completing tasks.", ephemeral=True) +@lightbulb.implements(lightbulb.SlashCommand, lightbulb.PrefixCommand) +async def tutorial(ctx: lightbulb.Context) -> None: + """Help for the bot.""" + await ctx.respond(tutorial_message(True, True)) + + +def load(bot: lightbulb.BotApp): + """Add the plugin to the bot.""" + bot.add_plugin(plugin) + + +def unload(bot: lightbulb.BotApp): + """Remove the plugin to the bot.""" + bot.remove_plugin(plugin) diff --git a/discord-bot/bot/messages.py b/discord-bot/bot/messages.py index c1a6d355..7bd57bb9 100644 --- a/discord-bot/bot/messages.py +++ b/discord-bot/bot/messages.py @@ -73,6 +73,10 @@ def _hint(hint: str | None) -> str: return f"{NL}Hint: {hint}" if hint else "" +def _li(text: str) -> str: + return f":small_blue_diamond: {text}" + + ### # Messages ### @@ -222,6 +226,60 @@ def confirm_ranking_response_message(content: str, items: list[str]) -> str: """ +def help_message(can_manage_guild: bool, is_dev: bool) -> str: + """The /help command message.""" + content = f"""\ +{_h1("HELP")} + +{_li("**`/help`**")} +Show this message. + +{_li("**`/work [type]`**")} +Start a new task. +**`[type]`**: +The type of task to start. If not provided, a random task will be selected. The different types are +:small_orange_diamond: `random`: A random task type +:small_orange_diamond: ~~`summarize_story`~~ (coming soon) +:small_orange_diamond: ~~`rate_summary`~~ (coming soon) +:small_orange_diamond: `initial_prompt`: Ask the assistant something +:small_orange_diamond: `prompter_reply`: Reply to the assistant +:small_orange_diamond: `assistant_reply`: Reply to the user +:small_orange_diamond: `rank_initial_prompts`: Rank some initial prompts +:small_orange_diamond: `rank_prompter_replies`: Rank some prompter replies +:small_orange_diamond: `rank_assistant_replies`: Rank some assistant replies + +To learn how to complete tasks, run `/tutorial`. +""" + if can_manage_guild: + content += f"""\ + +{_li("**`/settings log_channel `**")} +Set the channel that the bot logs completed task messages in. +**``**: The channel to log completed tasks in. The bot needs to be able to send messages in this channel. + +{_li("**`/settings get`**")} +Get the current settings. +""" + if is_dev: + content += f"""\ + +{_li("**`/reload [plugin]`**")} +Hot-reload a plugin. Only code *inside* of function bodies will be updated. +Any changes to __function signatures__, __other files__, __decorators__, or __imports__ will require a restart. +**`[plugin]`**: +The plugin to hot-reload. If no plugin is provided, all plugins are hot-reload. +""" + return content + + +def tutorial_message() -> str: + """The /tutorial command message.""" + # TODO: Finish message + return f"""\ +{_h1("TUTORIAL")} +""" + + def confirm_label_response_message(content: str) -> str: user_labels = content.lower().replace(" ", "").split(",") user_labels_str = ", ".join(user_labels) From 88ee3b32644ee3f8a249dc281150f86505490b5e Mon Sep 17 00:00:00 2001 From: Sotirios Anagnostidis Date: Fri, 6 Jan 2023 21:28:26 +0100 Subject: [PATCH 24/32] merge deepspeed --- model/supervised_finetuning/README.md | 35 +++- .../supervised_finetuning/configs/config.yaml | 19 +- .../custom_datasets/__init__.py | 10 +- .../custom_datasets/prompt_dialogue.py | 66 +++++++ model/supervised_finetuning/losses.py | 2 +- model/supervised_finetuning/trainer.py | 171 ++++++------------ model/supervised_finetuning/utils.py | 10 +- 7 files changed, 181 insertions(+), 132 deletions(-) create mode 100644 model/supervised_finetuning/custom_datasets/prompt_dialogue.py diff --git a/model/supervised_finetuning/README.md b/model/supervised_finetuning/README.md index 014afa95..bd202397 100644 --- a/model/supervised_finetuning/README.md +++ b/model/supervised_finetuning/README.md @@ -23,7 +23,40 @@ open-asisstant dataset are available it will be added here. ## Model -TBD +Normally you should be able to add new models in configs/config.yml + +``` +your-model-name: + learning_rate: 2e-6 + model_name: + weight_decay: 0.01 + max_length: 812 + warmup_steps: 600 + gradient_checkpointing: false + gradient_accumulation_steps: 5 + per_device_train_batch_size: 4 + per_device_eval_batch_size: 4 +``` + +``` +python trainer.py --configs defaults your-model-name +``` + +However, if the model of your choice doesn't have pad_token, eos_token, +sep_token, you have to update utils.py `get_tokenizer` to use the right token. + +## Deepspeed support + +You can edit the configs/zero_config.json and use any stage you wish. The +current config uses zero-stage 3. For more details on how to setup the config +checkout [this page](https://www.deepspeed.ai/tutorials/zero/) + +Once you are satisfy with your deepzero config, you can add --deepspeed flag at +the end to trigger deepspeed + +``` +python trainer.py --configs defaults your-model-name --deepspeed +``` ## Results diff --git a/model/supervised_finetuning/configs/config.yaml b/model/supervised_finetuning/configs/config.yaml index 64c024c2..fb2bdaa0 100644 --- a/model/supervised_finetuning/configs/config.yaml +++ b/model/supervised_finetuning/configs/config.yaml @@ -6,7 +6,7 @@ defaults: per_device_eval_batch_size: 2 weight_decay: 0.00 warmup_steps: 600 - eval_steps: 200 + eval_steps: 100 save_steps: 500 max_length: 512 num_train_epochs: 3 @@ -17,6 +17,7 @@ defaults: freeze_layer: datasets: - webgpt + - prompt_dialogue cache_dir: ~/.cache loss_fn: CrossEntropyLoss eval_size: @@ -44,11 +45,21 @@ gpt-jt: per_device_train_batch_size: 4 per_device_eval_batch_size: 4 +codegen: + learning_rate: 2e-6 + model_name: Salesforce/codegen-2B-multi + weight_decay: 0.01 + max_length: 812 + warmup_steps: 600 + gradient_checkpointing: false + gradient_accumulation_steps: 5 + per_device_train_batch_size: 4 + per_device_eval_batch_size: 4 + debug: eval_steps: 20 eval_size: 100 - model_name: EleutherAI/gpt-j-6B - gradient_accumulation_steps: 2 + gradient_accumulation_steps: 1 per_device_train_batch_size: 1 per_device_eval_batch_size: 1 - quantization: 8bit \ No newline at end of file + quantization: \ No newline at end of file diff --git a/model/supervised_finetuning/custom_datasets/__init__.py b/model/supervised_finetuning/custom_datasets/__init__.py index 7e3bdc79..5706bfa7 100644 --- a/model/supervised_finetuning/custom_datasets/__init__.py +++ b/model/supervised_finetuning/custom_datasets/__init__.py @@ -2,6 +2,8 @@ from datasets import load_dataset from sklearn.model_selection import train_test_split from torch.utils.data import Dataset, Subset +from .prompt_dialogue import PromptGeneratedDataset + QA_SPECIAL_TOKENS = {"Question": "", "Answer": ""} @@ -14,8 +16,8 @@ class SquadV2Dataset(Dataset): def __getitem__(self, idx): data = self.dataset[idx] - # dummy return first answer - return "".join([data["title"], ". ", data["context"], " " + data["question"]]), data["answers"]["text"][0] + # return first answer form list of possible answers + return data["title"] + ". " + data["context"] + " " + data["question"], data["answers"]["text"][0] class WebGPT(Dataset): @@ -57,12 +59,14 @@ def get_one_dataset(conf, dataset_name): dataset_name = dataset_name.lower() if dataset_name == "squadv2": - raise ValueError("SquadV2 is not diverse enough for generation .. ") train = SquadV2Dataset(conf.cache_dir, "train") eval = SquadV2Dataset(conf.cache_dir, "validation") elif dataset_name == "webgpt": dataset = WebGPT() train, eval = train_val_dataset(dataset, val_split=0.2) + elif dataset_name == "prompt_dialogue": + dataset = PromptGeneratedDataset() + train, eval = train_val_dataset(dataset, val_split=0.2) else: raise ValueError(f"Unknown dataset {dataset_name}") diff --git a/model/supervised_finetuning/custom_datasets/prompt_dialogue.py b/model/supervised_finetuning/custom_datasets/prompt_dialogue.py new file mode 100644 index 00000000..17911141 --- /dev/null +++ b/model/supervised_finetuning/custom_datasets/prompt_dialogue.py @@ -0,0 +1,66 @@ +import os +from urllib.request import urlopen + +from torch.utils.data import Dataset + + +class PromptGeneratedDataset(Dataset): + """Generates from flan 11B + User: What are the best methods for preventing a slave trade? + + Rosey: The best methods .... + <|endoftext|> + + we are ignoring results with multiple lines for now + """ + + url = "https://github.com/Rallio67/language-model-agents/raw/main/chat_dialogue_v2_c.txt" + + def __init__(self) -> None: + super().__init__() + os.makedirs("datasets", exist_ok=True) + chat_dialogue = os.path.join("datasets", "chat_dialogue_v2_c.txt") + if not os.path.exists(chat_dialogue): + with urlopen(self.url) as file: + content = file.read().decode() + with open(chat_dialogue, "w") as fout: + fout.write(content) + + question = "" + answer = "" + self.pairs = [] + with open(chat_dialogue, "r") as f: + corpus = f.read().split("<|endoftext|>") + for dialogue in corpus: + dialogue = dialogue.strip() + if "Rosey:" in dialogue: + user, bot = dialogue.split("Rosey:", maxsplit=1) + question = user.split(":", maxsplit=1)[1].strip() + answer = bot.strip() + if len(answer) and len(question): + self.pairs.append((question, answer)) + + if len(question) > 0 and len(answer) > 0: + self.pairs.append((question, answer)) + + def __len__(self): + return len(self.pairs) + + def __getitem__(self, index): + question, answer = self.pairs[index] + return question, answer + + +if __name__ == "__main__": + from torch.utils.data import DataLoader + from transformers import AutoTokenizer + + from .dialogue_collator import DialogueDataCollator + + tokenizer = AutoTokenizer.from_pretrained("Salesforce/codegen-2B-multi") + tokenizer.add_special_tokens({"pad_token": "<|endoftext|>", "sep_token": "<|endoftext|>"}) + dataset = PromptGeneratedDataset() + collate_fn = DialogueDataCollator(tokenizer, padding=True, max_length=128) + dataloader = DataLoader(dataset, collate_fn=collate_fn, batch_size=5) + for batch in dataloader: + print(batch["input_ids"].shape) diff --git a/model/supervised_finetuning/losses.py b/model/supervised_finetuning/losses.py index 795396b9..0cc639cf 100644 --- a/model/supervised_finetuning/losses.py +++ b/model/supervised_finetuning/losses.py @@ -7,7 +7,7 @@ class CrossEntropyLoss(nn.CrossEntropyLoss): def forward(self, input, target, mask=None): if mask is not None: - mask = mask.view(-1) + mask = mask.view(-1).bool() input = input.view(-1, input.size(-1)) target = target.view(-1) input = input[mask] diff --git a/model/supervised_finetuning/trainer.py b/model/supervised_finetuning/trainer.py index 590f9bbd..bb77b9c3 100644 --- a/model/supervised_finetuning/trainer.py +++ b/model/supervised_finetuning/trainer.py @@ -1,34 +1,19 @@ import argparse import os -from dataclasses import dataclass from distutils.util import strtobool -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import torch from torch import nn from torch.utils.data import Dataset -from transformers import ( - DataCollator, - EvalPrediction, - PreTrainedModel, - PreTrainedTokenizerBase, - Trainer, - TrainerCallback, - TrainingArguments, - get_cosine_schedule_with_warmup, -) +from transformers import PreTrainedModel, Trainer, TrainingArguments, get_cosine_schedule_with_warmup import bitsandbytes as bnb + from utils import get_dataset, get_loss, get_model, get_tokenizer, read_yamls os.environ["WANDB_PROJECT"] = "supervised-finetuning" -@dataclass -class CustomTrainingArguments(TrainingArguments): - loss_function: str = "CrossEntropyLoss" - quantization: str = None - - def compute_metrics(eval_pred): pred_ids = eval_pred.predictions labels = eval_pred.label_ids @@ -46,35 +31,15 @@ class SFTTrainer(Trainer): self, model: Union[PreTrainedModel, nn.Module] = None, args: TrainingArguments = None, - data_collator: Optional[DataCollator] = None, - train_dataset: Optional[Dataset] = None, - eval_dataset: Optional[Dataset] = None, - tokenizer: Optional[PreTrainedTokenizerBase] = None, - model_init: Callable[[], PreTrainedModel] = None, - compute_metrics: Optional[Callable[[EvalPrediction], Dict]] = None, - callbacks: Optional[List[TrainerCallback]] = None, - optimizers: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None), - preprocess_logits_for_metrics: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = None, + loss_function: str = "CrossEntropyLoss", + **kwargs, ): - super().__init__( - model, - args, - data_collator, - train_dataset, - eval_dataset, - tokenizer, - model_init, - compute_metrics, - callbacks, - optimizers, - preprocess_logits_for_metrics, - ) + super().__init__(model, args, **kwargs) # By default CrossEntropyLoss ignores padding_index -100, but just in case use our own loss_fct - self.loss_fct = get_loss(args.loss_function) + self.loss_fct = get_loss(loss_function) def create_optimizer_and_scheduler(self, num_training_steps: int): - print("Optimizer") if self.args.quantization == "8bit": self.optimizer = bnb.optim.Adam8bit(model.parameters(), lr=0.001, betas=(0.9, 0.995)) else: @@ -82,7 +47,6 @@ class SFTTrainer(Trainer): self.model.parameters(), lr=self.args.learning_rate, weight_decay=self.args.weight_decay ) - print("lr sheduler") self.lr_scheduler = get_cosine_schedule_with_warmup( self.optimizer, num_warmup_steps=self.args.warmup_steps, @@ -93,16 +57,17 @@ class SFTTrainer(Trainer): def compute_loss(self, model, inputs, return_outputs=False): labels_mask = inputs.pop("label_masks") + targets = inputs.pop("targets") outputs = model(**inputs) - loss = self.loss_fct(outputs.get("logits"), torch.roll(inputs["input_ids"], -1, -1), mask=labels_mask) + loss = self.loss_fct(outputs.get("logits"), targets, mask=labels_mask) return (loss, outputs) if return_outputs else loss def _compute_loss(self, model, inputs): - labels_mask = inputs.pop("label_masks") + targets = inputs.pop("targets") inputs = self._prepare_inputs(inputs) @@ -110,7 +75,6 @@ class SFTTrainer(Trainer): logits = outputs.get("logits") - targets = torch.roll(inputs["input_ids"], -1, -1) loss = self.loss_fct(outputs.get("logits"), targets, mask=labels_mask) return loss, logits, targets, labels_mask @@ -142,12 +106,18 @@ def _strtobool(x): def argument_parsing(notebook=False, notebook_args=None): parser = argparse.ArgumentParser() parser.add_argument("--configs", nargs="+", required=True) + parser.add_argument("--local_rank", type=int, default=-1) + parser.add_argument("--deepspeed", action="store_true") + parser.add_argument("--no-deepspeed", dest="deepspeed", action="store_false") + parser.set_defaults(deepspeed=False) if notebook: args, remaining = parser.parse_known_args(notebook_args) else: args, remaining = parser.parse_known_args() + print(args) + # Config from YAML conf = {} configs = read_yamls("./configs") @@ -158,6 +128,8 @@ def argument_parsing(notebook=False, notebook_args=None): else: conf.update(configs[name]) + conf["local_rank"] = args.local_rank + conf["deepspeed"] = args.deepspeed # Override config from command-line parser = argparse.ArgumentParser() for key, value in conf.items(): @@ -175,76 +147,41 @@ if __name__ == "__main__": tokenizer = get_tokenizer(training_conf) model = get_model(training_conf, tokenizer) - ### - from datasets import load_dataset - from bitsandbytes.optim import Adam8bit - from torch.nn import functional as F - from tqdm import tqdm + train, evals, collate_fn = get_dataset(training_conf, tokenizer) - gpt = model.to("cuda") + args = TrainingArguments( + output_dir=f"{training_conf.model_name}-{training_conf.log_dir}-finetuned", + num_train_epochs=training_conf.num_train_epochs, + warmup_steps=training_conf.warmup_steps, + learning_rate=float(training_conf.learning_rate), + deepspeed="configs/zero_config.json" if training_conf.deepspeed else None, + fp16=True, + local_rank=training_conf.local_rank, + gradient_checkpointing=training_conf.gradient_checkpointing, + gradient_accumulation_steps=training_conf.gradient_accumulation_steps, + per_device_train_batch_size=training_conf.per_device_train_batch_size, + per_device_eval_batch_size=training_conf.per_device_eval_batch_size, + weight_decay=training_conf.weight_decay, + max_grad_norm=training_conf.max_grad_norm, + logging_steps=training_conf.logging_steps, + save_total_limit=training_conf.save_total_limit, + evaluation_strategy="steps", + eval_steps=training_conf.eval_steps, + save_steps=training_conf.save_steps, + eval_accumulation_steps=training_conf.eval_accumulation_steps, + report_to="wandb", + ) - gpt.gradient_checkpointing_enable() - - codeparrot = load_dataset("transformersbook/codeparrot-train", streaming=True, cache_dir=training_conf.cache_dir) - optimizer = Adam8bit(gpt.parameters(), lr=1e-5) - - with torch.cuda.amp.autocast(): - for row in tqdm(codeparrot["train"]): - if len(row["content"]) <= 1: - continue - - batch = tokenizer(row["content"], truncation=True, max_length=128, return_tensors="pt") - batch = {k: v.cuda() for k, v in batch.items()} - - out = gpt.forward( - **batch, - ) - - loss = F.cross_entropy( - out.logits[:, :-1, :].flatten(0, -2), batch["input_ids"][:, 1:].flatten(), reduction="mean" - ) - print(loss) - loss.backward() - - optimizer.step() - optimizer.zero_grad() - ### - - - # train, evals, collate_fn = get_dataset(training_conf, tokenizer) - # assert len(evals) > 0 - - # args = CustomTrainingArguments( - # output_dir=f"{training_conf.model_name}-{training_conf.log_dir}-finetuned", - # num_train_epochs=training_conf.num_train_epochs, - # warmup_steps=training_conf.warmup_steps, - # loss_function=training_conf.loss_fn, - # learning_rate=float(training_conf.learning_rate), - # fp16=True, - # gradient_checkpointing=training_conf.gradient_checkpointing, - # gradient_accumulation_steps=training_conf.gradient_accumulation_steps, - # per_device_train_batch_size=training_conf.per_device_train_batch_size, - # per_device_eval_batch_size=training_conf.per_device_eval_batch_size, - # weight_decay=training_conf.weight_decay, - # max_grad_norm=training_conf.max_grad_norm, - # logging_steps=training_conf.logging_steps, - # save_total_limit=training_conf.save_total_limit, - # evaluation_strategy="steps", - # eval_steps=training_conf.eval_steps, - # save_steps=training_conf.save_steps, - # eval_accumulation_steps=training_conf.eval_accumulation_steps, - # report_to="wandb", - # quantization=training_conf.quantization, - # ) - - # trainer = SFTTrainer( - # model, - # args, - # train_dataset=train, - # eval_dataset=evals, - # data_collator=collate_fn, - # tokenizer=tokenizer, - # compute_metrics=compute_metrics, - # preprocess_logits_for_metrics=preprocess_logits_for_metrics, - # ) - # trainer.train() + assert len(evals) > 0 + trainer = SFTTrainer( + model, + args, + loss_function=training_conf.loss_fn, + train_dataset=train, + eval_dataset=evals, + data_collator=collate_fn, + tokenizer=tokenizer, + compute_metrics=compute_metrics, + preprocess_logits_for_metrics=preprocess_logits_for_metrics, + ) + trainer.train() diff --git a/model/supervised_finetuning/utils.py b/model/supervised_finetuning/utils.py index c3cc512c..8f9ed5ca 100644 --- a/model/supervised_finetuning/utils.py +++ b/model/supervised_finetuning/utils.py @@ -15,6 +15,10 @@ def get_tokenizer(conf): if "galactica" in conf.model_name: tokenizer.add_special_tokens({"pad_token": "", "eos_token": ""}) + elif "GPT-JT" in conf.model_name: + tokenizer.add_special_tokens({"pad_token": tokenizer.eos_token, "sep_token": "<|extratoken_100|>"}) + elif "codegen" in conf.model_name: + tokenizer.add_special_tokens({"pad_token": "<|endoftext|>", "sep_token": "<|endoftext|>"}) additional_special_tokens = ( [] @@ -29,12 +33,6 @@ def get_tokenizer(conf): def get_model(conf, tokenizer): - if not any([x in conf.model_name.lower() for x in SUPPORTED_MODELS]): - raise ValueError( - f"Model {conf.model_name} not supported. Supported models: {SUPPORTED_MODELS}. " - "To include more make sure the masking is done correctly... (decoder only supported for now)" - ) - model = get_specific_model(conf.model_name, conf.cache_dir, conf.quantization) if len(tokenizer) != model.get_input_embeddings().num_embeddings: From 148244455cd29e77cdd3967f44502cab329396f3 Mon Sep 17 00:00:00 2001 From: Sotirios Anagnostidis Date: Fri, 6 Jan 2023 21:29:38 +0100 Subject: [PATCH 25/32] refactor --- .../custom_datasets/dialogue_collator.py | 36 +++++++++---------- model/supervised_finetuning/models/gptj.py | 2 +- model/supervised_finetuning/requirements.txt | 3 ++ 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/model/supervised_finetuning/custom_datasets/dialogue_collator.py b/model/supervised_finetuning/custom_datasets/dialogue_collator.py index f9e1bb5e..479931f6 100644 --- a/model/supervised_finetuning/custom_datasets/dialogue_collator.py +++ b/model/supervised_finetuning/custom_datasets/dialogue_collator.py @@ -24,26 +24,24 @@ class DialogueDataCollator: flatten_messages = [] label_masks = [] - for messages in features: - assert len(messages) % 2 == 0, "Number of messages must be even" + for feature_one in features: + assert len(feature_one) % 2 == 0, "Number of messages must be even" messages = [ (QA_SPECIAL_TOKENS["Question"] if i % 2 == 0 else "") + x + (QA_SPECIAL_TOKENS["Answer"] if i % 2 == 0 else "") - for i, x in enumerate(messages) + for i, x in enumerate(feature_one) ] # Add a way for the model to terminate generation # When we predict the start of a new expected question, we want to be able to stop generation messages.append(QA_SPECIAL_TOKENS["Question"]) - flatten_messages.append( - self.tokenizer( - "".join(messages), - truncation=True, - max_length=self.max_length, - return_offsets_mapping=True, - ) + flatten_message = self.tokenizer( + "".join(messages), + truncation=True, + max_length=self.max_length, + return_offsets_mapping=True, ) message_change_indices = np.cumsum([len(x) for x in messages[:-1]]) @@ -57,18 +55,19 @@ class DialogueDataCollator: message_indices = list( map( lambda x: next((i for i, val in enumerate(message_change_indices) if val >= x), -2), - list(map(lambda x: x[1], flatten_messages[-1]["offset_mapping"])), + list(map(lambda x: x[1], flatten_message["offset_mapping"])), ) ) label_mask = np.roll(list(map(lambda x: x % 2 == 1, message_indices)), -1, -1) try: label_mask[[i for i in range(len(message_indices)) if message_indices[i] == -2][0] - 1] = True except IndexError: - # an aftermath of padding - pass + # due to truncation, we might not have the last termination token + label_mask[-1] = False label_masks.append(label_mask) - flatten_messages[-1].pop("offset_mapping") + + flatten_messages.append({k: v for k, v in flatten_message.items() if k != "offset_mapping"}) batch = self.tokenizer.pad( flatten_messages, @@ -79,10 +78,9 @@ class DialogueDataCollator: ) dim = batch["input_ids"].shape[-1] - batch["label_masks"] = torch.stack([F.pad(torch.tensor(x), (0, dim - len(x))) for x in label_masks]) - - for k in list(batch.keys()): - if k not in ["input_ids", "attention_mask", "label_masks"]: - batch.pop(k) + batch["label_masks"] = torch.stack( + [F.pad(torch.tensor(x), (0, dim - len(x)), value=False) for x in label_masks] + ) + batch["targets"] = torch.roll(batch["input_ids"], -1, -1) return batch diff --git a/model/supervised_finetuning/models/gptj.py b/model/supervised_finetuning/models/gptj.py index 3cbec3ce..d954c830 100644 --- a/model/supervised_finetuning/models/gptj.py +++ b/model/supervised_finetuning/models/gptj.py @@ -171,7 +171,7 @@ def get_model(model_name, cache_dir, quantization): if quantization is None: model = AutoModelForCausalLM.from_pretrained(model_name, cache_dir=cache_dir) elif quantization == "8bit": - print("Loading 8-bit model") + raise ValueError("Loading 8-bit model. Bitsandbytes does not behave so far...") transformers.models.gptj.modeling_gptj.GPTJBlock = GPTJBlock model = AutoModelForCausalLM.from_pretrained(model_name, cache_dir=cache_dir) add_adapters(model) diff --git a/model/supervised_finetuning/requirements.txt b/model/supervised_finetuning/requirements.txt index d579468f..798b5950 100644 --- a/model/supervised_finetuning/requirements.txt +++ b/model/supervised_finetuning/requirements.txt @@ -4,3 +4,6 @@ PyYAML==6.0 scikit_learn==1.2.0 torch==1.13.1 transformers==4.25.1 +deepspeed==0.7.7 +mpi4py==3.1.4 +accelerate==0.15.0 \ No newline at end of file From 44d05ed709b4fae5aec32c72f0b29bbcdf6d3f83 Mon Sep 17 00:00:00 2001 From: klotske Date: Sat, 7 Jan 2023 00:06:15 +0300 Subject: [PATCH 26/32] Initial implementation of progressbar tracking --- .../src/components/Survey/TrackedTextarea.tsx | 35 +++++++++++++++++++ website/src/pages/create/assistant_reply.tsx | 24 +++++++++---- website/src/pages/create/initial_prompt.tsx | 22 +++++++++--- website/src/pages/create/summarize_story.tsx | 22 +++++++++--- website/src/pages/create/user_reply.tsx | 22 +++++++++--- 5 files changed, 104 insertions(+), 21 deletions(-) create mode 100644 website/src/components/Survey/TrackedTextarea.tsx diff --git a/website/src/components/Survey/TrackedTextarea.tsx b/website/src/components/Survey/TrackedTextarea.tsx new file mode 100644 index 00000000..205d7588 --- /dev/null +++ b/website/src/components/Survey/TrackedTextarea.tsx @@ -0,0 +1,35 @@ +import { Progress, Stack, Textarea, TextareaProps } from "@chakra-ui/react"; + +interface TrackedTextboxProps { + text: string; + thresholds: { + low: number; + medium: number; + goal: number; + }; + textareaProps?: TextareaProps; + onTextChange: (event: React.ChangeEvent) => void; +} + +export const TrackedTextarea = (props: TrackedTextboxProps) => { + const wordCount = props.text.split(" ").length - 1; + + let progressColor: string; + switch (true) { + case wordCount < props.thresholds.low: + progressColor = "red"; + break; + case wordCount < props.thresholds.medium: + progressColor = "yellow"; + break; + default: + progressColor = "green"; + } + + return ( + +