diff --git a/src/peft/tuners/adalora.py b/src/peft/tuners/adalora.py
index fc6261f..36c67bd 100644
--- a/src/peft/tuners/adalora.py
+++ b/src/peft/tuners/adalora.py
@@ -117,11 +117,11 @@ class AdaLoraModel(LoraModel):
                 "When using multiple adapters, set inference_mode to True for all adapters except the one you want to train."
             )
 
+        mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias)
         if self.peft_config[adapter_name].inference_mode:
             _freeze_adapter(self.model, adapter_name)
         else:
             self.trainable_adapter_name = adapter_name
-            mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias)
             self.rankallocator = RankAllocator(self.model, self.peft_config[adapter_name], self.trainable_adapter_name)
 
     def _find_and_replace(self, adapter_name):
diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py
index 90e4a23..cd07e45 100644
--- a/src/peft/tuners/lora.py
+++ b/src/peft/tuners/lora.py
@@ -143,10 +143,9 @@ class LoraModel(torch.nn.Module):
             raise ValueError(
                 "LoraModel supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters."
             )
+        mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias)
         if self.peft_config[adapter_name].inference_mode:
             _freeze_adapter(self.model, adapter_name)
-        else:
-            mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias)
 
     def _find_and_replace(self, adapter_name):
         lora_config = self.peft_config[adapter_name]