From 7b7038273a08577da74260a4cb28a98cb2eafc5e Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Sat, 8 Apr 2023 11:32:00 +0530 Subject: [PATCH] fix trainable params issue --- src/peft/tuners/adalora.py | 2 +- src/peft/tuners/lora.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/peft/tuners/adalora.py b/src/peft/tuners/adalora.py index fc6261f..36c67bd 100644 --- a/src/peft/tuners/adalora.py +++ b/src/peft/tuners/adalora.py @@ -117,11 +117,11 @@ class AdaLoraModel(LoraModel): "When using multiple adapters, set inference_mode to True for all adapters except the one you want to train." ) + mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias) if self.peft_config[adapter_name].inference_mode: _freeze_adapter(self.model, adapter_name) else: self.trainable_adapter_name = adapter_name - mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias) self.rankallocator = RankAllocator(self.model, self.peft_config[adapter_name], self.trainable_adapter_name) def _find_and_replace(self, adapter_name): diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py index 90e4a23..cd07e45 100644 --- a/src/peft/tuners/lora.py +++ b/src/peft/tuners/lora.py @@ -143,10 +143,9 @@ class LoraModel(torch.nn.Module): raise ValueError( "LoraModel supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters." ) + mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias) if self.peft_config[adapter_name].inference_mode: _freeze_adapter(self.model, adapter_name) - else: - mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias) def _find_and_replace(self, adapter_name): lora_config = self.peft_config[adapter_name]