Merge pull request #17 from huggingface/smangrul/fixes

add examples and update README
2026-09-10 12:20:21 +08:00 · 2022-12-29 17:34:56 +05:30
parent 89675315cb f16750afcc
commit 866891c6c4
10 changed files with 5218 additions and 89 deletions
@@ -18,18 +18,13 @@ Supported methods:

 ```python
 from transformers import AutoModelForSeq2SeqLM
-from pet import get_pet_config, get_pet_model
+from pet import get_pet_config, get_pet_model, LoRAConfig, TaskType
 model_name_or_path = "bigscience/mt0-large"
 tokenizer_name_or_path = "bigscience/mt0-large"

-config = {
-    "pet_type":"LORA",
-    "task_type":"SEQ_2_SEQ_LM",
-    "r": 8,
-    "lora_alpha": 32,
-    "lora_dropout": 0.1
-}
-pet_config = get_pet_config(config)
+pet_config = LoRAConfig(
+    task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1
+)

 model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)
 model = get_pet_model(model, pet_config)
@@ -73,9 +68,9 @@ Save storage by avoiding full finetuning of models on each of the downstream tas
 With PET methods, users only need to store tiny checkpoints in the order of `MBs` all the while retaining 
 performance comparable to full finetuning.

-An example of using LoRA for the task of adaping `LayoutLMForTokenClassification` on `FUNSD` dataset is given in `~examples/PET_LoRA_LayoutLMForTokenClassification_on_FUNSD.py`. We can observe that with only `0.62 %` of parameters being trainable, we achieve performance (F1 0.777) comparable to full finetuning (F1 0.786) (without any hyerparam tuning runs for extracting more performance), and the checkpoint of this is only `2.8MB`.
+An example of using LoRA for the task of adaping `LayoutLMForTokenClassification` on `FUNSD` dataset is given in `~examples/token_classification/PET_LoRA_LayoutLMForTokenClassification_on_FUNSD.py`. We can observe that with only `0.62 %` of parameters being trainable, we achieve performance (F1 0.777) comparable to full finetuning (F1 0.786) (without any hyerparam tuning runs for extracting more performance), and the checkpoint of this is only `2.8MB`. Now, if there are `N` such datasets, just have these PET models one for each dataset and save a lot of storage without having to worry about the problem of catastrophic forgetting or overfitting of backbone/base model.

-Now, if there are `N` such datasets, just have these PET models one for each dataset and save a lot of storage without having to worry about the problem of catastrophic forgetting or overfitting of backbone/base model.
+Another example is fine-tuning `roberta-large` on `MRPC` GLUE dataset suing differenct PET methods. The notebooks are given in `~examples/sequence_classification`. 


 ## PET + 🤗 Accelerate
@@ -83,9 +78,66 @@ Now, if there are `N` such datasets, just have these PET models one for each dat
 PET models work with 🤗 Accelerate out of the box. Use 🤗 Accelerate for Distributed training on various hardware such as GPUs, Apple Silicon devices etc during training.
 Use 🤗 Accelerate for inferencing on consumer hardware with small resources.

-### Example of PET model distributed training using 🤗 Accelerate
+### Example of PET model training using 🤗 Accelerate's DeepSpeed integation

-### Example of PET model inference using 🤗 Accelerate
+ Currently DeepSpeed requires PR [ZeRO3 handling frozen weights](https://github.com/microsoft/DeepSpeed/pull/2653) to fix [[REQUEST] efficiently deal with frozen weights during training](https://github.com/microsoft/DeepSpeed/issues/2615) issue. Example is provided in `~examples/conditional_generation/pet_lora_seq2seq_accelerate_ds_zero3_offload.py`. 
+  a. First run `accelerate config --config_file ds_zero3_cpu.yaml` and answer the questionaire. 
+  Below are the contents of the config file.
+  ```
+  compute_environment: LOCAL_MACHINE
+  deepspeed_config:
+    gradient_accumulation_steps: 1
+    gradient_clipping: 1.0
+    offload_optimizer_device: cpu
+    offload_param_device: cpu
+    zero3_init_flag: true
+    zero3_save_16bit_model: true
+    zero_stage: 3
+  distributed_type: DEEPSPEED
+  downcast_bf16: 'no'
+  dynamo_backend: 'NO'
+  fsdp_config: {}
+  machine_rank: 0
+  main_training_function: main
+  megatron_lm_config: {}
+  mixed_precision: 'no'
+  num_machines: 1
+  num_processes: 1
+  rdzv_backend: static
+  same_network: true
+  use_cpu: false
+  ```
+  b. run the below command to launch example script
+  ```
+  accelerate launch --config_file ds_zero3_cpu.yaml examples/pet_lora_seq2seq_accelerate_ds_zero3_offload.py
+  ```
+
+  c. output logs:
+  ```bash
+  GPU Memory before entering the train : 1916
+  GPU Memory consumed at the end of the train (end-begin): 66
+  GPU Peak Memory consumed during the train (max-begin): 7488
+  GPU Total Peak Memory consumed during the train (max): 9404
+  CPU Memory before entering the train : 19411
+  CPU Memory consumed at the end of the train (end-begin): 0
+  CPU Peak Memory consumed during the train (max-begin): 0
+  CPU Total Peak Memory consumed during the train (max): 19411
+  epoch=4: train_ppl=tensor(1.0705, device='cuda:0') train_epoch_loss=tensor(0.0681, device='cuda:0')
+  100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:27<00:00,  3.92s/it]
+  GPU Memory before entering the eval : 1982
+  GPU Memory consumed at the end of the eval (end-begin): -66
+  GPU Peak Memory consumed during the eval (max-begin): 672
+  GPU Total Peak Memory consumed during the eval (max): 2654
+  CPU Memory before entering the eval : 19411
+  CPU Memory consumed at the end of the eval (end-begin): 0
+  CPU Peak Memory consumed during the eval (max-begin): 0
+  CPU Total Peak Memory consumed during the eval (max): 19411
+  accuracy=100.0
+  eval_preds[:10]=['no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint', 'no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint']
+  dataset['train'][label_column][:10]=['no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint', 'no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint']
+  ```
+
+### Example of PET model inference using 🤗 Accelerate's Big Model Inferencing capabilities


 ## Models support matrix
@@ -134,39 +186,7 @@ Use 🤗 Accelerate for inferencing on consumer hardware with small resources.

 ## Caveats:

-1. Currently DeepSpeed requires PR [ZeRO3 handling frozen weights](https://github.com/microsoft/DeepSpeed/pull/2653) to fix [[REQUEST] efficiently deal with frozen weights during training](https://github.com/microsoft/DeepSpeed/issues/2615) issue on DeepSpeed repository. Example is provided in `~examples/pet_lora_seq2seq_accelerate_ds_zero3_offload.py`. 
-  a. First run `accelerate config --config_file ds_zero3_cpu.yaml` and answer the questionaire. 
-  Below are the contents of the config file.
-  ```
-  compute_environment: LOCAL_MACHINE
-  deepspeed_config:
-    gradient_accumulation_steps: 1
-    gradient_clipping: 1.0
-    offload_optimizer_device: cpu
-    offload_param_device: cpu
-    zero3_init_flag: true
-    zero3_save_16bit_model: true
-    zero_stage: 3
-  distributed_type: DEEPSPEED
-  downcast_bf16: 'no'
-  dynamo_backend: 'NO'
-  fsdp_config: {}
-  machine_rank: 0
-  main_training_function: main
-  megatron_lm_config: {}
-  mixed_precision: 'no'
-  num_machines: 1
-  num_processes: 1
-  rdzv_backend: static
-  same_network: true
-  use_cpu: false
-  ```
-  b. run the below command to launch example script
-  ```
-  accelerate launch --config_file ds_zero3_cpu.yaml examples/pet_lora_seq2seq_accelerate_ds_zero3_offload.py
-  ```
-
-2. Below is an example of using PyTorch FSDP for training. However, it doesn't lead to 
+1. Below is an example of using PyTorch FSDP for training. However, it doesn't lead to 
 any GPU memory savings. Please refer issue [[FSDP] FSDP with CPU offload consumes 1.65X more GPU memory when training models with most of the params frozen](https://github.com/pytorch/pytorch/issues/91165). 

  ```python
@@ -180,7 +200,7 @@ any GPU memory savings. Please refer issue [[FSDP] FSDP with CPU offload consume
  model = accelerator.prepare(model)
  ```

-  Example of parameter efficient tuning with `mt0-xxl` base model using 🤗 Accelerate is provided in `~examples/pet_lora_seq2seq_accelerate_fsdp.py`. 
+  Example of parameter efficient tuning with `mt0-xxl` base model using 🤗 Accelerate is provided in `~examples/conditional_generation/pet_lora_seq2seq_accelerate_fsdp.py`. 
  a. First run `accelerate config --config_file fsdp_config.yaml` and answer the questionaire. 
  Below are the contents of the config file.
  ```
@@ -218,9 +238,9 @@ any GPU memory savings. Please refer issue [[FSDP] FSDP with CPU offload consume
  accelerate launch --config_file fsdp_config.yaml examples/pet_lora_seq2seq_accelerate_fsdp.py
  ```

-3. When using `P_TUNING` or `PROMPT_TUNING` with `SEQ_2_SEQ` task, remember to remove the `num_virtual_token` virtual prompt predictions from the left side of the model outputs during evaluations. 
+2. When using `P_TUNING` or `PROMPT_TUNING` with `SEQ_2_SEQ` task, remember to remove the `num_virtual_token` virtual prompt predictions from the left side of the model outputs during evaluations. 

-4. `P_TUNING` or `PROMPT_TUNING` doesn't support `generate` functionality of transformers bcause `generate` strictly requires `input_ids`/`decoder_input_ids` but 
+3. `P_TUNING` or `PROMPT_TUNING` doesn't support `generate` functionality of transformers bcause `generate` strictly requires `input_ids`/`decoder_input_ids` but 
 `P_TUNING`/`PROMPT_TUNING` appends soft prompt embeddings to `input_embeds` to create
 new `input_embeds` to be given to the model. Therefore, `generate` doesn't support this yet.

@@ -8,7 +8,7 @@
   "outputs": [],
   "source": [
    "from transformers import AutoModelForSeq2SeqLM\n",
-    "from pet import get_pet_config,get_pet_model, get_pet_model_state_dict\n",
+    "from pet import get_pet_config,get_pet_model, get_pet_model_state_dict, LoRAConfig, TaskType\n",
    "import torch\n",
    "from datasets import load_dataset\n",
    "import os\n",
@@ -23,13 +23,6 @@
    "model_name_or_path = \"bigscience/mt0-large\"\n",
    "tokenizer_name_or_path = \"bigscience/mt0-large\"\n",
    "\n",
-    "config = {\n",
-    "    \"pet_type\":\"LORA\",\n",
-    "    \"task_type\":\"SEQ_2_SEQ_LM\",\n",
-    "    \"r\":16,\n",
-    "    \"lora_alpha\": 32,\n",
-    "    \"lora_dropout\": 0.1\n",
-    "}\n",
    "checkpoint_name = \"financial_sentiment_analysis_lora_v1.pt\"\n",
    "text_column = \"sentence\"\n",
    "label_column = \"text_label\"\n",
@@ -47,7 +40,9 @@
   "outputs": [],
   "source": [
    "# creating model\n",
-    "pet_config = get_pet_config(config)\n",
+    "pet_config = LoRAConfig(\n",
+    "    task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1\n",
+    ")\n",
    "\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)\n",
    "model = get_pet_model(model, pet_config)\n",
@@ -402,7 +397,7 @@
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
-   "version": "3.10.5"
+   "version": "3.10.5 (v3.10.5:f377153967, Jun  6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]"
  },
  "vscode": {
   "interpreter": {
@@ -104,14 +104,14 @@ def main():
    accelerator = Accelerator()
    model_name_or_path = "bigscience/T0_3B"
    dataset_name = "twitter_complaints"
-    pet_config = pet_config = LoRAConfig(
-        task_type=TaskType.TOKEN_CLS, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1, bias="all"
+    pet_config = LoRAConfig(
+        task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1
    )
    checkpoint_name = f"{dataset_name}_{pet_config.pet_type}_{pet_config.task_type}_v1.pt".replace("/", "_")
    text_column = "Tweet text"
    label_column = "text_label"
    lr = 3e-3
-    num_epochs = 20
+    num_epochs = 5
    batch_size = 8
    seed = 42
    set_seed(seed)
@@ -178,11 +178,15 @@ def main():
        num_training_steps=(len(train_dataloader) * num_epochs),
    )

-    model, train_dataloader, eval_dataloader, optimizer, lr_scheduler = accelerator.prepare(
-        model, train_dataloader, eval_dataloader, optimizer, lr_scheduler
+    model, train_dataloader, eval_dataloader, test_dataloader, optimizer, lr_scheduler = accelerator.prepare(
+        model, train_dataloader, eval_dataloader, test_dataloader, optimizer, lr_scheduler
    )
    accelerator.print(model)

+    is_ds_zero_3 = False
+    if getattr(accelerator.state, "deepspeed_plugin", None):
+        is_ds_zero_3 = accelerator.state.deepspeed_plugin.zero_stage == 3
+
    for epoch in range(num_epochs):
        with TorchTracemalloc() as tracemalloc:
            model.train()
@@ -213,6 +217,9 @@ def main():
                tracemalloc.cpu_peaked + b2mb(tracemalloc.cpu_begin)
            )
        )
+        train_epoch_loss = total_loss / len(eval_dataloader)
+        train_ppl = torch.exp(train_epoch_loss)
+        accelerator.print(f"{epoch=}: {train_ppl=} {train_epoch_loss=}")

        model.eval()
        eval_preds = []
@@ -220,12 +227,11 @@ def main():
            for _, batch in enumerate(tqdm(eval_dataloader)):
                batch = {k: v for k, v in batch.items() if k != "labels"}
                with torch.no_grad():
-                    outputs = model.generate(**batch, synced_gpus=True)  # synced_gpus=True for DS-stage 3
+                    outputs = accelerator.unwrap_model(model).generate(
+                        **batch, synced_gpus=is_ds_zero_3
+                    )  # synced_gpus=True for DS-stage 3
                preds = outputs.detach().cpu().numpy()
                eval_preds.extend(tokenizer.batch_decode(preds, skip_special_tokens=True))
-            train_epoch_loss = total_loss / len(eval_dataloader)
-            train_ppl = torch.exp(train_epoch_loss)
-            accelerator.print(f"{epoch=}: {train_ppl=} {train_epoch_loss=}")

        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
        accelerator.print("GPU Memory before entering the eval : {}".format(b2mb(tracemalloc.begin)))
@@ -248,23 +254,23 @@ def main():

        correct = 0
        total = 0
-        for pred, true in zip(eval_preds, dataset["validation"][label_column]):
+        for pred, true in zip(eval_preds, dataset["train"][label_column]):
            if pred.strip() == true.strip():
                correct += 1
            total += 1
        accuracy = correct / total * 100
        accelerator.print(f"{accuracy=}")
        accelerator.print(f"{eval_preds[:10]=}")
-        accelerator.print(f"{dataset['validation'][label_column][:10]=}")
-    accelerator.wait_for_everyone()
-    accelerator.save(get_pet_model_state_dict(model, state_dict=accelerator.get_state_dict(model)), checkpoint_name)
-    accelerator.wait_for_everyone()
+        accelerator.print(f"{dataset['train'][label_column][:10]=}")

    model.eval()
    test_preds = []
    for _, batch in enumerate(tqdm(test_dataloader)):
        batch = {k: v for k, v in batch.items() if k != "labels"}
-        outputs = model.generate(**batch, synced_gpus=True)  # synced_gpus=True for DS-stage 3
+        with torch.no_grad():
+            outputs = accelerator.unwrap_model(model).generate(
+                **batch, synced_gpus=is_ds_zero_3
+            )  # synced_gpus=True for DS-stage 3
        test_preds.extend(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))

    test_preds_cleaned = []
@@ -272,16 +278,20 @@ def main():
        test_preds_cleaned.append(get_closest_label(pred, classes))

    test_df = dataset["test"].to_pandas()
-    test_df["text_labels"] = test_preds_cleaned
+    test_df[label_column] = test_preds_cleaned
    test_df["text_labels_orig"] = test_preds
-    accelerator.print(test_df.sample(20))
+    accelerator.print(test_df[[text_column, label_column]].sample(20))

-    pred_df = test_df[["ID", "text_labels"]]
+    pred_df = test_df[["ID", label_column]]
    pred_df.columns = ["ID", "Label"]

    os.makedirs(f"data/{dataset_name}", exist_ok=True)
    pred_df.to_csv(f"data/{dataset_name}/predictions.csv", index=False)

+    accelerator.wait_for_everyone()
+    accelerator.save(get_pet_model_state_dict(model, state_dict=accelerator.get_state_dict(model)), checkpoint_name)
+    accelerator.wait_for_everyone()
+

 if __name__ == "__main__":
    main()
@@ -6,7 +6,7 @@ from torch.utils.data import DataLoader
 from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, default_data_collator, get_linear_schedule_with_warmup

 from datasets import load_dataset
-from pet import get_pet_config, get_pet_model, get_pet_model_state_dict
+from pet import LoRAConfig, TaskType, get_pet_model, get_pet_model_state_dict
 from pet.utils.other import fsdp_auto_wrap_policy
 from tqdm import tqdm

@@ -22,8 +22,9 @@ def main():
    num_epochs = 1
    base_path = "temp/data/FinancialPhraseBank-v1.0"

-    config = {"pet_type": "LORA", "task_type": "SEQ_2_SEQ_LM", "r": 8, "lora_alpha": 32, "lora_dropout": 0.1}
-    pet_config = get_pet_config(config)
+    pet_config = LoRAConfig(
+        task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1
+    )
    checkpoint_name = "financial_sentiment_analysis_lora_fsdp_v1.pt"
    model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)
    model = get_pet_model(model, pet_config)
@@ -125,7 +126,9 @@ def main():
        accelerator.print(f"{eval_preds[:10]=}")
        accelerator.print(f"{dataset['validation'][label_column][:10]=}")
        accelerator.wait_for_everyone()
-        accelerator.save(get_pet_model_state_dict(model), checkpoint_name)
+        accelerator.save(
+            get_pet_model_state_dict(model, state_dict=accelerator.get_state_dict(model)), checkpoint_name
+        )
        accelerator.wait_for_everyone()


@@ -8,7 +8,7 @@
   "outputs": [],
   "source": [
    "from transformers import AutoModelForSeq2SeqLM\n",
-    "from pet import get_pet_config,get_pet_model, get_pet_model_state_dict\n",
+    "from pet import get_pet_config,get_pet_model, get_pet_model_state_dict, PrefixTuningConfig, TaskType\n",
    "import torch\n",
    "from datasets import load_dataset\n",
    "import os\n",
@@ -24,11 +24,6 @@
    "model_name_or_path = \"t5-large\"\n",
    "tokenizer_name_or_path = \"t5-large\"\n",
    "\n",
-    "config = {\n",
-    "    \"pet_type\":\"PREFIX_TUNING\",\n",
-    "    \"task_type\":\"SEQ_2_SEQ_LM\",\n",
-    "    \"num_virtual_tokens\": 20\n",
-    "}\n",
    "checkpoint_name = \"financial_sentiment_analysis_prefix_tuning_v1.pt\"\n",
    "text_column = \"sentence\"\n",
    "label_column = \"text_label\"\n",
@@ -46,7 +41,9 @@
   "outputs": [],
   "source": [
    "# creating model\n",
-    "pet_config = get_pet_config(config)\n",
+    "pet_config =  PrefixTuningConfig(\n",
+    "    task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, num_virtual_tokens=20\n",
+    ")\n",
    "\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)\n",
    "model = get_pet_model(model, pet_config)\n",
@@ -492,7 +489,7 @@
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
-   "version": "3.10.5"
+   "version": "3.10.5 (v3.10.5:f377153967, Jun  6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]"
  },
  "vscode": {
   "interpreter": {