From 510f172c58ff200d9c74eba636058c1dd7c8da56 Mon Sep 17 00:00:00 2001 From: QingruZhang Date: Wed, 1 Mar 2023 21:26:07 +0000 Subject: [PATCH 1/6] adalora example --- .../peft_lora_seq2seq.ipynb | 2 +- src/peft/mapping.py | 2 +- src/peft/peft_model.py | 4 ++- src/peft/tuners/adalora.py | 26 +++++++++---------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/examples/conditional_generation/peft_lora_seq2seq.ipynb b/examples/conditional_generation/peft_lora_seq2seq.ipynb index f22d3c6..bf864d4 100644 --- a/examples/conditional_generation/peft_lora_seq2seq.ipynb +++ b/examples/conditional_generation/peft_lora_seq2seq.ipynb @@ -473,7 +473,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.16" }, "vscode": { "interpreter": { diff --git a/src/peft/mapping.py b/src/peft/mapping.py index afc8bbe..ceb220e 100644 --- a/src/peft/mapping.py +++ b/src/peft/mapping.py @@ -20,7 +20,7 @@ from .peft_model import ( PeftModelForSequenceClassification, PeftModelForTokenClassification, ) -from .tuners import LoraConfig, AdaLoraConfig PrefixTuningConfig, PromptEncoderConfig, PromptTuningConfig +from .tuners import LoraConfig, AdaLoraConfig, PrefixTuningConfig, PromptEncoderConfig, PromptTuningConfig from .utils import PromptLearningConfig diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index 4703059..ed92162 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -29,7 +29,7 @@ from transformers.utils import PushToHubMixin from huggingface_hub import hf_hub_download -from .tuners import LoraModel, PrefixEncoder, PromptEmbedding, PromptEncoder +from .tuners import LoraModel, AdaLoraConfig, AdaLoraModel, PrefixEncoder, PromptEmbedding, PromptEncoder from .utils import ( TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING, WEIGHTS_NAME, @@ -76,6 +76,8 @@ class PeftModel(PushToHubMixin, torch.nn.Module): self.modules_to_save = None if isinstance(self.peft_config, PromptLearningConfig): self._setup_prompt_encoder() + elif isinstance(self.peft_config, AdaLoraConfig): + self.base_model = AdaLoraModel(peft_config, model) else: self.base_model = LoraModel(peft_config, model) if getattr(self.peft_config, "modules_to_save", None) is not None: diff --git a/src/peft/tuners/adalora.py b/src/peft/tuners/adalora.py index f53df06..9405d92 100644 --- a/src/peft/tuners/adalora.py +++ b/src/peft/tuners/adalora.py @@ -13,7 +13,7 @@ import torch.nn.functional as F from transformers.pytorch_utils import Conv1D from ..utils import PeftConfig, PeftType, transpose -from .lora import LoraConfig, LoraModel, LoRALayer, mark_only_lora_as_trainable +from .lora import LoraConfig, LoraModel, LoraLayer, mark_only_lora_as_trainable def is_bnb_available(): @@ -69,26 +69,26 @@ class AdaLoraModel(LoraModel): Args: model ([`transformers.PreTrainedModel`]): The model to be adapted. - config ([`LoraConfig`]): The configuration of the Lora model. + config ([`AdaLoraConfig`]): The configuration of the AdaLora model. Returns: - `torch.nn.Module`: The Lora model. + `torch.nn.Module`: The AdaLora model. Example:: - >>> from transformers import AutoModelForSeq2SeqLM, LoraConfig >>> from peft import LoraModel, LoraConfig >>> - config = LoraConfig( - peft_type="LORA", task_type="SEQ_2_SEQ_LM", r=8, lora_alpha=32, target_modules=["q", "v"], - lora_dropout=0.01, ) - >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> lora_model = LoraModel(config, model) + >>> from transformers import AutoModelForSeq2SeqLM, LoraConfig >>> from peft import AdaLoraModel, AdaLoraConfig + >>> config = AdaLoraConfig( + peft_type="ADALORA", task_type="SEQ_2_SEQ_LM", r=8, lora_alpha=32, target_modules=["q", "v"], + lora_dropout=0.01, + ) + >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> adalora_model = AdaLoraModel(config, model) **Attributes**: - **model** ([`transformers.PreTrainedModel`]) -- The model to be adapted. - - **peft_config** ([`LoraConfig`]): The configuration of the Lora model. + - **peft_config** ([`AdaLoraConfig`]): The configuration of the AdaLora model. """ def __init__(self, config, model): - # super().__init__() nn.Module.__init__(self) self.peft_config = config self.model = model @@ -194,9 +194,7 @@ class AdaLoraModel(LoraModel): - - -class SVDLinear(nn.Linear, LoRALayer): +class SVDLinear(nn.Linear, LoraLayer): # SVD-based adaptation for a dense layer def __init__( self, @@ -210,7 +208,7 @@ class SVDLinear(nn.Linear, LoRALayer): **kwargs ): nn.Linear.__init__(self, in_features, out_features, **kwargs) - LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, + LoraLayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights) self.fan_in_fan_out = fan_in_fan_out From 1a3680d8a74ae677a49d3d89daa203bc384f26af Mon Sep 17 00:00:00 2001 From: QingruZhang Date: Wed, 1 Mar 2023 21:52:33 +0000 Subject: [PATCH 2/6] test for adalora example --- .../peft_adalora_seq2seq.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/examples/conditional_generation/peft_adalora_seq2seq.py b/examples/conditional_generation/peft_adalora_seq2seq.py index 19875c4..5163f65 100644 --- a/examples/conditional_generation/peft_adalora_seq2seq.py +++ b/examples/conditional_generation/peft_adalora_seq2seq.py @@ -1,5 +1,5 @@ from transformers import AutoModelForSeq2SeqLM -from peft import get_peft_config, get_peft_model, get_peft_model_state_dict, LoraConfig, TaskType +from peft import get_peft_config, get_peft_model, get_peft_model_state_dict, LoraConfig, AdaLoraConfig, AdaLoraModel, TaskType import torch from datasets import load_dataset import os @@ -12,20 +12,24 @@ from tqdm import tqdm from datasets import load_dataset device = "cuda" -model_name_or_path = "bigscience/mt0-large" -tokenizer_name_or_path = "bigscience/mt0-large" +model_name_or_path = "facebook/bart-base" +tokenizer_name_or_path = "facebook/bart-base" checkpoint_name = "financial_sentiment_analysis_lora_v1.pt" text_column = "sentence" label_column = "text_label" max_length = 128 lr = 1e-3 -num_epochs = 3 +num_epochs = 1 batch_size = 8 # creating model -peft_config = LoraConfig(task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1) +peft_config = AdaLoraConfig( + r=8, lora_alpha=32, lora_dropout=0.1 + task_type=TaskType.SEQ_2_SEQ_LM, + inference_mode=False +) model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path) model = get_peft_model(model, peft_config) @@ -89,6 +93,7 @@ lr_scheduler = get_linear_schedule_with_warmup( num_warmup_steps=0, num_training_steps=(len(train_dataloader) * num_epochs), ) +model.base_model.peft_config.total_step = len(train_dataloader) * num_epochs # training and evaluation From 35cd771c975df2469c37261aaec6344c265a9691 Mon Sep 17 00:00:00 2001 From: QingruZhang Date: Wed, 1 Mar 2023 21:55:31 +0000 Subject: [PATCH 3/6] example --- examples/conditional_generation/peft_adalora_seq2seq.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/conditional_generation/peft_adalora_seq2seq.py b/examples/conditional_generation/peft_adalora_seq2seq.py index 5163f65..ec6b91a 100644 --- a/examples/conditional_generation/peft_adalora_seq2seq.py +++ b/examples/conditional_generation/peft_adalora_seq2seq.py @@ -26,7 +26,7 @@ batch_size = 8 # creating model peft_config = AdaLoraConfig( - r=8, lora_alpha=32, lora_dropout=0.1 + r=8, lora_alpha=32, lora_dropout=0.1, task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False ) @@ -151,7 +151,7 @@ model.save_pretrained(peft_model_id) ckpt = f"{peft_model_id}/adapter_model.bin" -get_ipython().system('du -h $ckpt') +# get_ipython().system('du -h $ckpt') from peft import PeftModel, PeftConfig From 7471035885a4ff19081a77a98da9735bb33db75c Mon Sep 17 00:00:00 2001 From: QingruZhang Date: Thu, 2 Mar 2023 01:04:48 +0000 Subject: [PATCH 4/6] finish the testing and debugging --- .../peft_adalora_seq2seq.py | 25 ++++-- src/peft/peft_model.py | 2 + src/peft/tuners/adalora.py | 89 ++++++++----------- src/peft/utils/save_and_load.py | 4 +- 4 files changed, 58 insertions(+), 62 deletions(-) diff --git a/examples/conditional_generation/peft_adalora_seq2seq.py b/examples/conditional_generation/peft_adalora_seq2seq.py index ec6b91a..b3626f7 100644 --- a/examples/conditional_generation/peft_adalora_seq2seq.py +++ b/examples/conditional_generation/peft_adalora_seq2seq.py @@ -26,7 +26,10 @@ batch_size = 8 # creating model peft_config = AdaLoraConfig( - r=8, lora_alpha=32, lora_dropout=0.1, + init_r=12, target_r=1, + beta1=0.85, beta2=0.85, + tinit=0, tfinal=230, deltaT=1, + lora_alpha=32, lora_dropout=0.1, task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False ) @@ -98,19 +101,23 @@ model.base_model.peft_config.total_step = len(train_dataloader) * num_epochs # training and evaluation model = model.to(device) - +global_step = 0 for epoch in range(num_epochs): model.train() total_loss = 0 for step, batch in enumerate(tqdm(train_dataloader)): batch = {k: v.to(device) for k, v in batch.items()} - outputs = model(**batch) - loss = outputs.loss - total_loss += loss.detach().float() - loss.backward() - optimizer.step() - lr_scheduler.step() - optimizer.zero_grad() + with torch.autograd.set_detect_anomaly(True): + outputs = model(**batch) + loss = outputs.loss + total_loss += loss.detach().float() + loss.backward() + optimizer.step() + lr_scheduler.step() + + model.base_model.update_and_allocate(global_step) + optimizer.zero_grad() + global_step += 1 model.eval() eval_loss = 0 diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index ed92162..a3df6f6 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -180,6 +180,8 @@ class PeftModel(PushToHubMixin, torch.nn.Module): hook = AlignDevicesHook(io_same_device=True) if model.peft_config.peft_type == PeftType.LORA: add_hook_to_module(model.base_model.model, hook) + elif model.peft_config.peft_type == PeftType.ADALORA: + add_hook_to_module(model.base_model.model, hook) else: remove_hook_from_submodules(model.prompt_encoder) add_hook_to_module(model.base_model, hook) diff --git a/src/peft/tuners/adalora.py b/src/peft/tuners/adalora.py index 9405d92..ad91f6a 100644 --- a/src/peft/tuners/adalora.py +++ b/src/peft/tuners/adalora.py @@ -30,17 +30,15 @@ class AdaLoraConfig(LoraConfig): This is the configuration class to store the configuration of a [`~peft.AdaLora`]. Args: - r (`int`): Lora attention dimension - target_modules (`Union[List[str],str]`): The names of the modules to apply Lora to. - lora_alpha (`float`): The alpha parameter for Lora scaling. - lora_dropout (`float`): The dropout probability for Lora layers. - merge_weights (`bool`): - Whether to merge the weights of the Lora layers with the base transformer model in `eval` mode. - fan_in_fan_out (`bool`): Set this to True if the layer to replace stores weight like (fan_in, fan_out) - enable_lora ( `List[bool]`): Used with `lora.MergedLinear`. - bias (`str`): Bias type for Lora. Can be 'none', 'all' or 'lora_only' - modules_to_save (`List[str]`):List of modules apart from LoRA layers to be set as trainable - and saved in the final checkpoint. + target_r (`int`): The target average rank of incremental matrix. + init_r (`int`): The initial rank for each incremental matrix. + tinit (`int`): The steps of initial fine-tuning warmup. + tfinal (`int`): The step of final fine-tuning. + deltaT (`int`): The time internval between two budget allocations. + beta1 (`float`): The hyperparameter of EMA for sensitivity smoothing. + beta2 (`float`): The hyperparameter of EMA for undertainty quantification. + orth_reg_weight (`float`): The coefficient of orthogonal regularization. + total_step (`int`): The total training steps that should be specified before training. """ target_r: int = field(default=8, metadata={"help": "Target Lora matrix dimension."}) init_r: int = field(default=12, metadata={"help": "Intial Lora matrix dimension."}) @@ -81,7 +79,8 @@ class AdaLoraModel(LoraModel): peft_type="ADALORA", task_type="SEQ_2_SEQ_LM", r=8, lora_alpha=32, target_modules=["q", "v"], lora_dropout=0.01, ) - >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> adalora_model = AdaLoraModel(config, model) + >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") + >>> model = AdaLoraModel(config, model) **Attributes**: - **model** ([`transformers.PreTrainedModel`]) -- The model to be adapted. @@ -94,7 +93,6 @@ class AdaLoraModel(LoraModel): self.model = model self._find_and_replace() mark_only_lora_as_trainable(self.model, self.peft_config.bias) - # self.forward = self.model.forward self.rankallocator = RankAllocator(config, self.named_parameters()) def _find_and_replace(self): @@ -173,7 +171,7 @@ class AdaLoraModel(LoraModel): assert orth_reg_weight > 0 if hasattr(outputs, "loss"): - regu_loss = None + regu_loss = 0 num_param = 0 for n,p in self.model.named_parameters(): if "lora_A" in n or "lora_B" in n: @@ -181,12 +179,9 @@ class AdaLoraModel(LoraModel): I = torch.eye(*para_cov.size(), out=torch.empty_like(para_cov)) I.requires_grad = False num_param += 1 - if regu_loss is None: - regu_loss = torch.norm(para_cov-I, p="fro") - else: - regu_loss += torch.norm(para_cov-I, p="fro") - - outputs.loss += orth_reg_weight * regu_loss + regu_loss += torch.norm(para_cov-I, p="fro") + regu_loss = regu_loss / num_param + outputs.loss += orth_reg_weight * regu_loss return outputs def update_and_allocate(self, global_step): @@ -195,7 +190,7 @@ class AdaLoraModel(LoraModel): class SVDLinear(nn.Linear, LoraLayer): - # SVD-based adaptation for a dense layer + # SVD-based adaptation by a dense layer def __init__( self, in_features: int, @@ -239,8 +234,6 @@ class SVDLinear(nn.Linear, LoraLayer): nn.init.normal_(self.lora_B, mean=0.0, std=0.02) def train(self, mode: bool = True): - # def T(w): - # return w.T if self.fan_in_fan_out else w nn.Linear.train(self, mode) if self.merge_weights and self.merged: # Make sure that the weights are not merged @@ -251,8 +244,6 @@ class SVDLinear(nn.Linear, LoraLayer): self.merged = False def eval(self): - # def T(w): - # return w.T if self.fan_in_fan_out else w nn.Linear.eval(self) if self.merge_weights and not self.merged: # Merge the weights and mark it @@ -263,8 +254,6 @@ class SVDLinear(nn.Linear, LoraLayer): self.merged = True def forward(self, x: torch.Tensor): - # def T(w): - # return w.T if self.fan_in_fan_out else w if self.r > 0 and not self.merged: result = F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias) if self.r > 0: @@ -277,9 +266,8 @@ class SVDLinear(nn.Linear, LoraLayer): if is_bnb_available(): - class SVDLinear8bitLt(bnb.nn.Linear8bitLt, LoraLayer): - # Lora implemented in a dense layer + # Low-rank matrix for SVD-based adaptation def __init__( self, in_features, @@ -315,20 +303,11 @@ if is_bnb_available(): # Freezing the pre-trained weight matrix self.weight.requires_grad = False self.ranknum.requires_grad = False - - # self.lora_A = nn.Linear(in_features, r, bias=False) - # self.lora_B = nn.Linear(r, out_features, bias=False) - # self.scaling = self.lora_alpha / self.r - # # Freezing the pre-trained weight matrix - # self.weight.requires_grad = False self.reset_parameters() def reset_parameters(self): if hasattr(self, "lora_A"): # initialize A the same way as the default for nn.Linear and B to zero - # nn.init.kaiming_uniform_(self.lora_A.weight, a=math.sqrt(5)) - # nn.init.zeros_(self.lora_B.weight) - nn.init.zeros_(self.lora_E) nn.init.normal_(self.lora_A, mean=0.0, std=0.02) nn.init.normal_(self.lora_B, mean=0.0, std=0.02) @@ -347,13 +326,11 @@ if is_bnb_available(): output = ( self.lora_dropout(x) @ (self.lora_A*self.lora_E).T @ self.lora_B.T /(self.ranknum+1e-5) ).to(expected_dtype) * self.scaling - # output = self.lora_B(self.lora_A(self.lora_dropout(x))).to(expected_dtype) * self.scaling result += output else: output = ( self.lora_dropout(x) @ (self.lora_A*self.lora_E).T @ self.lora_B.T /(self.ranknum+1e-5) ) * self.scaling - # output = self.lora_B(self.lora_A(self.lora_dropout(x))) * self.scaling result += output return result @@ -400,8 +377,8 @@ class RankAllocator(object): if step <= tinit: budget = self.init_bgt mask_ind = False - # Final warmup - elif step > self.total_step - tfinal: + # Final fine-tuning + elif step > total_step - tfinal: budget = self.target_bgt mask_ind = True else: @@ -415,6 +392,7 @@ class RankAllocator(object): def update_ipt(self, model): + # Update the sensitivity and uncertainty for every weight for n,p in model.named_parameters(): if "lora_" in n: if n not in self.ipt: @@ -423,8 +401,10 @@ class RankAllocator(object): self.exp_avg_unc[n] = torch.zeros_like(p) with torch.no_grad(): self.ipt[n] = (p * p.grad).abs().detach() + # Sensitivity smoothing self.exp_avg_ipt[n] = self.beta1 * self.exp_avg_ipt[n] + \ (1 - self.beta1)*self.ipt[n] + # Uncertainty quantification self.exp_avg_unc[n] = self.beta2 * self.exp_avg_unc[n] + \ (1-self.beta2)*(self.ipt[n]-self.exp_avg_ipt[n]).abs() @@ -443,29 +423,31 @@ class RankAllocator(object): value_ipt = {} vector_ipt = {} triplet_ipt = {} + # Get the importance score for A, E, B for n,p in model.named_parameters(): if "lora_A" in n: - ipt_score = self._element_score(n) - comb_ipt = torch.mean(ipt_score, dim=1, keepdim=True) + entry_ipt = self._element_score(n) + comb_ipt = torch.mean(entry_ipt, dim=1, keepdim=True) name_m = n.replace("lora_A", "%s") if name_m not in vector_ipt: vector_ipt[name_m] = [comb_ipt] else: vector_ipt[name_m].append(comb_ipt) if "lora_B" in n: - ipt_score = self._element_score(n) - comb_ipt = torch.mean(ipt_score, dim=0, keepdim=False).view(-1, 1) + entry_ipt = self._element_score(n) + comb_ipt = torch.mean(entry_ipt, dim=0, keepdim=False).view(-1, 1) name_m = n.replace("lora_B", "%s") if name_m not in vector_ipt: vector_ipt[name_m] = [comb_ipt] else: vector_ipt[name_m].append(comb_ipt) if "lora_E" in n: - ipt_score = self._element_score(n) + entry_ipt = self._element_score(n) name_m = n.replace("lora_E", "%s") - value_ipt[name_m] = ipt_score + value_ipt[name_m] = entry_ipt all_score = [] + # Calculate the score for each triplet for name_m in vector_ipt: ipt_E = value_ipt[name_m] ipt_AB = torch.cat(vector_ipt[name_m], dim=1) @@ -474,11 +456,13 @@ class RankAllocator(object): triplet_ipt[name_E] = sum_ipt.view(-1, 1) all_score.append(sum_ipt.view(-1)) + # Get the threshold by ranking ipt mask_threshold = torch.kthvalue( torch.cat(all_score), k = self.init_bgt - budget, )[0].item() + # Mask the unimportant triplets with torch.no_grad(): for n,p in model.named_parameters(): if "lora_E" in n: @@ -486,14 +470,17 @@ class RankAllocator(object): return mask_threshold def update_and_allocate(self, model, global_step): - if global_step < self.peft_config.total_step - self.tfinal: + # Update the importance score and allocate the budget + if global_step < self.peft_config.total_step - self.peft_config.tfinal: self.update_ipt(model) + # TODO: Finalize the budget distribution by replacing with new Linear. budget, mask_ind = self.budget_schedule(global_step) + print("budget:", budget) if mask_ind: - mask_threshold = self.mask_to_budget(model, budget) + mask_threshold = self.mask_to_budget(model, budget) + print("mask threshold:", mask_threshold) else: mask_threshold = None - return budget, mask_threshold diff --git a/src/peft/utils/save_and_load.py b/src/peft/utils/save_and_load.py index c6596c7..86e388b 100644 --- a/src/peft/utils/save_and_load.py +++ b/src/peft/utils/save_and_load.py @@ -29,7 +29,7 @@ def get_peft_model_state_dict(model, state_dict=None): """ if state_dict is None: state_dict = model.state_dict() - if model.peft_config.peft_type == PeftType.LORA: + if model.peft_config.peft_type in (PeftType.LORA, PeftType.ADALORA): # to_return = lora_state_dict(model, bias=model.peft_config.bias) # adapted from `https://github.com/microsoft/LoRA/blob/main/loralib/utils.py` # to directly with the state dict which is necessary when using DeepSpeed or FSDP @@ -72,7 +72,7 @@ def set_peft_model_state_dict(model, peft_model_state_dict): """ model.load_state_dict(peft_model_state_dict, strict=False) - if model.peft_config.peft_type != PeftType.LORA: + if model.peft_config.peft_type not in (PeftType.LORA, PeftType.ADALORA): model.prompt_encoder.embedding.load_state_dict( {"weight": peft_model_state_dict["prompt_embeddings"]}, strict=True ) From 0a0c6ea6eac9a0e6aa420abb55a7a0ed418cf442 Mon Sep 17 00:00:00 2001 From: QingruZhang Date: Thu, 2 Mar 2023 01:08:41 +0000 Subject: [PATCH 5/6] adalora training example --- .../peft_adalora_seq2seq.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/examples/conditional_generation/peft_adalora_seq2seq.py b/examples/conditional_generation/peft_adalora_seq2seq.py index b3626f7..49fa497 100644 --- a/examples/conditional_generation/peft_adalora_seq2seq.py +++ b/examples/conditional_generation/peft_adalora_seq2seq.py @@ -1,5 +1,5 @@ from transformers import AutoModelForSeq2SeqLM -from peft import get_peft_config, get_peft_model, get_peft_model_state_dict, LoraConfig, AdaLoraConfig, AdaLoraModel, TaskType +from peft import get_peft_model, AdaLoraConfig, AdaLoraModel, TaskType import torch from datasets import load_dataset import os @@ -20,15 +20,15 @@ text_column = "sentence" label_column = "text_label" max_length = 128 lr = 1e-3 -num_epochs = 1 +num_epochs = 8 batch_size = 8 # creating model peft_config = AdaLoraConfig( - init_r=12, target_r=1, + init_r=12, target_r=8, beta1=0.85, beta2=0.85, - tinit=0, tfinal=230, deltaT=1, + tinit=200, tfinal=1000, deltaT=10, lora_alpha=32, lora_dropout=0.1, task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False @@ -107,17 +107,17 @@ for epoch in range(num_epochs): total_loss = 0 for step, batch in enumerate(tqdm(train_dataloader)): batch = {k: v.to(device) for k, v in batch.items()} - with torch.autograd.set_detect_anomaly(True): - outputs = model(**batch) - loss = outputs.loss - total_loss += loss.detach().float() - loss.backward() - optimizer.step() - lr_scheduler.step() - - model.base_model.update_and_allocate(global_step) - optimizer.zero_grad() - global_step += 1 + outputs = model(**batch) + loss = outputs.loss + total_loss += loss.detach().float() + loss.backward() + optimizer.step() + lr_scheduler.step() + # Update the importance of low-rank matrices + # and allocate the budget accordingly. + model.base_model.update_and_allocate(global_step) + optimizer.zero_grad() + global_step += 1 model.eval() eval_loss = 0 From fa65b95b9e9613f3adb705af1151ea5823bc76a0 Mon Sep 17 00:00:00 2001 From: QingruZhang Date: Thu, 2 Mar 2023 01:11:14 +0000 Subject: [PATCH 6/6] update comment --- src/peft/tuners/adalora.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/peft/tuners/adalora.py b/src/peft/tuners/adalora.py index ad91f6a..61bd512 100644 --- a/src/peft/tuners/adalora.py +++ b/src/peft/tuners/adalora.py @@ -63,7 +63,8 @@ class AdaLoraConfig(LoraConfig): class AdaLoraModel(LoraModel): """ - Creates Adaptive LoRA (AdaLora) model from a pretrained transformers model. + Creates AdaLoRA (Adaptive LoRA) model from a pretrained transformers model. + Paper: https://openreview.net/pdf?id=lq62uWRJjiY Args: model ([`transformers.PreTrainedModel`]): The model to be adapted.