diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py
index 652f4aa..0280236 100644
--- a/src/peft/peft_model.py
+++ b/src/peft/peft_model.py
@@ -133,7 +133,7 @@ class PeftModel(PushToHubMixin, torch.nn.Module):
             peft_config.inference_mode = inference_mode
 
     @classmethod
-    def from_pretrained(cls, model, model_id, adapter_name="default", **kwargs):
+    def from_pretrained(cls, model, model_id, adapter_name="default", is_trainable=False, **kwargs):
         r"""
         Instantiate a [`LoraModel`] from a pretrained Lora configuration and weights.
 
@@ -160,6 +160,11 @@ class PeftModel(PushToHubMixin, torch.nn.Module):
         ) > 0:
             remove_hook_from_submodules(model)
 
+        if isinstance(config, PromptLearningConfig) and is_trainable:
+            raise ValueError("Cannot set a prompt learning adapter to trainable when loading pretrained adapter.")
+        else:
+            config.inference_mode = not is_trainable
+
         if config.task_type not in MODEL_TYPE_TO_PEFT_MODEL_MAPPING.keys():
             model = cls(model, config, adapter_name)
         else:
diff --git a/src/peft/tuners/adalora.py b/src/peft/tuners/adalora.py
index fc6261f..36c67bd 100644
--- a/src/peft/tuners/adalora.py
+++ b/src/peft/tuners/adalora.py
@@ -117,11 +117,11 @@ class AdaLoraModel(LoraModel):
                 "When using multiple adapters, set inference_mode to True for all adapters except the one you want to train."
             )
 
+        mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias)
         if self.peft_config[adapter_name].inference_mode:
             _freeze_adapter(self.model, adapter_name)
         else:
             self.trainable_adapter_name = adapter_name
-            mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias)
             self.rankallocator = RankAllocator(self.model, self.peft_config[adapter_name], self.trainable_adapter_name)
 
     def _find_and_replace(self, adapter_name):
diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py
index ed8bf9e..9af74c5 100644
--- a/src/peft/tuners/lora.py
+++ b/src/peft/tuners/lora.py
@@ -139,10 +139,9 @@ class LoraModel(torch.nn.Module):
             raise ValueError(
                 "LoraModel supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters."
             )
+        mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias)
         if self.peft_config[adapter_name].inference_mode:
             _freeze_adapter(self.model, adapter_name)
-        else:
-            mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias)
 
     def _find_and_replace(self, adapter_name):
         lora_config = self.peft_config[adapter_name]