diff --git a/tests/testing_common.py b/tests/testing_common.py
index cfe6cf2..38aee14 100644
--- a/tests/testing_common.py
+++ b/tests/testing_common.py
@@ -268,8 +268,12 @@ class PeftCommonTester:
 
             logits_transformers = transformers_model(**dummy_input)[0]
 
-            self.assertTrue(torch.allclose(logits_lora, logits_merged, atol=1e-4, rtol=1e-4))
-            self.assertFalse(torch.allclose(logits_merged, logits_transformers, atol=1e-10, rtol=1e-10))
+            if config.merge_weights:
+                self.assertTrue(torch.allclose(logits_lora, logits_merged, atol=1e-4, rtol=1e-4))
+                self.assertFalse(torch.allclose(logits_merged, logits_transformers, atol=1e-10, rtol=1e-10))
+            else:
+                self.assertFalse(torch.allclose(logits_lora, logits_merged, atol=1e-4, rtol=1e-4))
+                self.assertTrue(torch.allclose(logits_merged, logits_transformers, atol=1e-10, rtol=1e-10))
 
             with tempfile.TemporaryDirectory() as tmp_dirname:
                 model.save_pretrained(tmp_dirname)