diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py
index 45d88ec..4739dce 100644
--- a/src/peft/tuners/lora.py
+++ b/src/peft/tuners/lora.py
@@ -329,6 +329,7 @@ class LoraLayer:
             self.lora_B.update(nn.ModuleDict({adapter_name: nn.Linear(r, self.out_features, bias=False)}))
             self.scaling[adapter_name] = lora_alpha / r
         self.reset_lora_parameters(adapter_name)
+        self.to(self.weight.device)
 
     def reset_lora_parameters(self, adapter_name):
         if adapter_name in self.lora_A.keys():
diff --git a/src/peft/utils/save_and_load.py b/src/peft/utils/save_and_load.py
index 43680fe..fb4b252 100644
--- a/src/peft/utils/save_and_load.py
+++ b/src/peft/utils/save_and_load.py
@@ -86,6 +86,8 @@ def set_peft_model_state_dict(model, adapter_name, peft_model_state_dict):
                         key = key.replace(module_name, f"{module_name}.modules_to_save.{adapter_name}")
                         break
             state_dict[key] = value
+    else:
+        state_dict = peft_model_state_dict
 
     if config.peft_type == PeftType.LORA:
         peft_model_state_dict = {}
@@ -100,7 +102,6 @@ def set_peft_model_state_dict(model, adapter_name, peft_model_state_dict):
         peft_model_state_dict = state_dict
     else:
         raise NotImplementedError
-
     model.load_state_dict(peft_model_state_dict, strict=False)
     if isinstance(config, PromptLearningConfig):
         model.prompt_encoder[adapter_name].embedding.load_state_dict(