diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py index 51cd56f..ab5259d 100644 --- a/src/peft/tuners/lora.py +++ b/src/peft/tuners/lora.py @@ -270,11 +270,26 @@ class LoraModel(torch.nn.Module): # manually merge if not merged if not target.merged: # merge weights per: https://arxiv.org/pdf/2106.09685.pdf / page 4 - if target.r > 0: - target.weight.data += ( - transpose(target.lora_B.weight @ target.lora_A.weight, target.fan_in_fan_out) - * target.scaling - ).to(target.weight.dtype) + if isinstance(target, Linear): + if target.r > 0: + target.weight.data += ( + transpose(target.lora_B.weight @ target.lora_A.weight, target.fan_in_fan_out) + * target.scaling + ).to(target.weight.dtype) + else: + if target.r > 0: + delta_w = ( + F.conv1d( + target.lora_A.weight.data.unsqueeze(0), + target.lora_B.weight.data, + groups=sum(target.enable_lora), + ) + .squeeze(0) + .transpose(-2, -1) + ) + target.weight.data += transpose( + target.zero_pad(delta_w * target.scaling), not target.fan_in_fan_out + ).to(target.weight.dtype) target.merged = True self._replace_module(parent, target_name, new_module, target) diff --git a/tests/test_decoder_models.py b/tests/test_decoder_models.py new file mode 100644 index 0000000..f0a575e --- /dev/null +++ b/tests/test_decoder_models.py @@ -0,0 +1,85 @@ +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +import torch +from parameterized import parameterized +from transformers import AutoModelForCausalLM + +from .testing_common import PeftCommonTester, PeftTestConfigManager + + +PEFT_DECODER_MODELS_TO_TEST = [ + "hf-internal-testing/tiny-random-OPTForCausalLM", + "hf-internal-testing/tiny-random-GPTNeoXForCausalLM", + "hf-internal-testing/tiny-random-GPT2LMHeadModel", + "hf-internal-testing/tiny-random-BloomForCausalLM", + "hf-internal-testing/tiny-random-gpt_neo", + "hf-internal-testing/tiny-random-GPTJForCausalLM", +] + +FULL_GRID = { + "model_ids": PEFT_DECODER_MODELS_TO_TEST, + "task_type": "CAUSAL_LM", +} + + +class PeftDecoderModelTester(unittest.TestCase, PeftCommonTester): + r""" + Test if the PeftModel behaves as expected. This includes: + - test if the model has the expected methods + + We use parametrized.expand for debugging purposes to test each model individually. + """ + transformers_class = AutoModelForCausalLM + + def prepare_inputs_for_testing(self): + input_ids = torch.tensor([[1, 1, 1], [1, 2, 1]]).to(self.torch_device) + attention_mask = torch.tensor([[1, 1, 1], [1, 0, 1]]).to(self.torch_device) + + input_dict = { + "input_ids": input_ids, + "attention_mask": attention_mask, + } + + return input_dict + + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) + def test_attributes_parametrized(self, test_name, model_id, config_cls, config_kwargs): + self._test_model_attr(model_id, config_cls, config_kwargs) + + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) + def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls, config_kwargs): + self._test_prepare_for_training(model_id, config_cls, config_kwargs) + + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) + def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs): + self._test_save_pretrained(model_id, config_cls, config_kwargs) + + @parameterized.expand( + PeftTestConfigManager.get_grid_parameters( + { + "model_ids": PEFT_DECODER_MODELS_TO_TEST, + "lora_kwargs": {"init_lora_weights": [False], "merge_weights": [False, True]}, + "task_type": "CAUSAL_LM", + }, + ) + ) + def test_merge_layers(self, test_name, model_id, config_cls, config_kwargs): + self._test_merge_layers(model_id, config_cls, config_kwargs) + + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) + def test_generate(self, test_name, model_id, config_cls, config_kwargs): + self._test_generate(model_id, config_cls, config_kwargs) diff --git a/tests/test_encoder_decoder_models.py b/tests/test_encoder_decoder_models.py new file mode 100644 index 0000000..6cb5a36 --- /dev/null +++ b/tests/test_encoder_decoder_models.py @@ -0,0 +1,88 @@ +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +import torch +from parameterized import parameterized +from transformers import AutoModelForSeq2SeqLM + +from .testing_common import PeftCommonTester, PeftTestConfigManager + + +PEFT_ENCODER_DECODER_MODELS_TO_TEST = [ + "hf-internal-testing/tiny-random-T5ForConditionalGeneration", + "hf-internal-testing/tiny-random-BartForConditionalGeneration", +] + +FULL_GRID = {"model_ids": PEFT_ENCODER_DECODER_MODELS_TO_TEST, "task_type": "SEQ_2_SEQ_LM"} + + +def skip_non_lora_or_pt(test_list): + r""" + Skip tests that are not lora or prefix tuning + """ + return [test for test in test_list if ("lora" in test[0] or "prefix_tuning" in test[0])] + + +class PeftEncoderDecoderModelTester(unittest.TestCase, PeftCommonTester): + r""" + Test if the PeftModel behaves as expected. This includes: + - test if the model has the expected methods + + We use parametrized.expand for debugging purposes to test each model individually. + """ + transformers_class = AutoModelForSeq2SeqLM + + def prepare_inputs_for_testing(self): + input_ids = torch.tensor([[1, 1, 1], [1, 2, 1]]).to(self.torch_device) + decoder_input_ids = torch.tensor([[1, 1, 1], [1, 2, 1]]).to(self.torch_device) + attention_mask = torch.tensor([[1, 1, 1], [1, 0, 1]]).to(self.torch_device) + + input_dict = { + "input_ids": input_ids, + "decoder_input_ids": decoder_input_ids, + "attention_mask": attention_mask, + } + + return input_dict + + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) + def test_attributes_parametrized(self, test_name, model_id, config_cls, config_kwargs): + self._test_model_attr(model_id, config_cls, config_kwargs) + + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) + def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls, config_kwargs): + self._test_prepare_for_training(model_id, config_cls, config_kwargs) + + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) + def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs): + self._test_save_pretrained(model_id, config_cls, config_kwargs) + + @parameterized.expand( + PeftTestConfigManager.get_grid_parameters( + { + "model_ids": PEFT_ENCODER_DECODER_MODELS_TO_TEST, + "lora_kwargs": {"init_lora_weights": [False], "merge_weights": [True, False]}, + "task_type": "SEQ_2_SEQ_LM", + }, + ) + ) + def test_merge_layers(self, test_name, model_id, config_cls, config_kwargs): + self._test_merge_layers(model_id, config_cls, config_kwargs) + + # skip non lora models - generate does not work for prefix tuning, prompt tuning + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID, filter_params_func=skip_non_lora_or_pt)) + def test_generate(self, test_name, model_id, config_cls, config_kwargs): + self._test_generate(model_id, config_cls, config_kwargs) diff --git a/tests/test_peft_model.py b/tests/test_peft_model.py deleted file mode 100644 index 4280ff3..0000000 --- a/tests/test_peft_model.py +++ /dev/null @@ -1,238 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import tempfile -import unittest - -import torch -from parameterized import parameterized -from transformers import AutoModelForCausalLM - -from peft import ( - PeftModel, - get_peft_model, - get_peft_model_state_dict, - prepare_model_for_int8_training, -) - -from .testing_common import PeftTestConfigManager - - -PEFT_DECODER_MODELS_TO_TEST = [ - "hf-internal-testing/tiny-random-OPTForCausalLM", - "hf-internal-testing/tiny-random-GPTNeoXForCausalLM", - "hf-internal-testing/tiny-random-GPT2LMHeadModel", - "hf-internal-testing/tiny-random-BloomForCausalLM", - "hf-internal-testing/tiny-random-gpt_neo", - "hf-internal-testing/tiny-random-GPTJForCausalLM", -] - -FULL_GRID = { - "model_ids": PEFT_DECODER_MODELS_TO_TEST, -} - - -class PeftTestMixin: - torch_device = "cuda" if torch.cuda.is_available() else "cpu" - - -class PeftModelTester(unittest.TestCase, PeftTestMixin): - r""" - Test if the PeftModel behaves as expected. This includes: - - test if the model has the expected methods - - We use parametrized.expand for debugging purposes to test each model individually. - """ - - def _test_model_attr(self, model_id, config_cls, config_kwargs): - model = AutoModelForCausalLM.from_pretrained(model_id) - config = config_cls( - base_model_name_or_path=model_id, - **config_kwargs, - ) - model = get_peft_model(model, config) - - self.assertTrue(hasattr(model, "save_pretrained")) - self.assertTrue(hasattr(model, "from_pretrained")) - self.assertTrue(hasattr(model, "push_to_hub")) - - @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) - def test_attributes_parametrized(self, test_name, model_id, config_cls, config_kwargs): - self._test_model_attr(model_id, config_cls, config_kwargs) - - def _test_prepare_for_training(self, model_id, config_cls, config_kwargs): - model = AutoModelForCausalLM.from_pretrained(model_id).to(self.torch_device) - config = config_cls( - base_model_name_or_path=model_id, - **config_kwargs, - ) - model = get_peft_model(model, config) - - dummy_input = torch.LongTensor([[1, 1, 1]]).to(self.torch_device) - dummy_output = model.get_input_embeddings()(dummy_input) - - self.assertTrue(not dummy_output.requires_grad) - - # load with `prepare_model_for_int8_training` - model = AutoModelForCausalLM.from_pretrained(model_id).to(self.torch_device) - model = prepare_model_for_int8_training(model) - - for param in model.parameters(): - self.assertTrue(not param.requires_grad) - - config = config_cls( - base_model_name_or_path=model_id, - **config_kwargs, - ) - model = get_peft_model(model, config) - - # For backward compatibility - if hasattr(model, "enable_input_require_grads"): - model.enable_input_require_grads() - else: - - def make_inputs_require_grad(module, input, output): - output.requires_grad_(True) - - model.get_input_embeddings().register_forward_hook(make_inputs_require_grad) - - dummy_input = torch.LongTensor([[1, 1, 1]]).to(self.torch_device) - dummy_output = model.get_input_embeddings()(dummy_input) - - self.assertTrue(dummy_output.requires_grad) - - @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) - def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls, config_kwargs): - self._test_prepare_for_training(model_id, config_cls, config_kwargs) - - def _test_save_pretrained(self, model_id, config_cls, config_kwargs): - model = AutoModelForCausalLM.from_pretrained(model_id) - config = config_cls( - base_model_name_or_path=model_id, - **config_kwargs, - ) - model = get_peft_model(model, config) - model = model.to(self.torch_device) - - with tempfile.TemporaryDirectory() as tmp_dirname: - model.save_pretrained(tmp_dirname) - - model_from_pretrained = AutoModelForCausalLM.from_pretrained(model_id) - model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname) - - # check if the state dicts are equal - state_dict = get_peft_model_state_dict(model) - state_dict_from_pretrained = get_peft_model_state_dict(model_from_pretrained) - - # check if same keys - self.assertEqual(state_dict.keys(), state_dict_from_pretrained.keys()) - - # check if tensors equal - for key in state_dict.keys(): - self.assertTrue( - torch.allclose( - state_dict[key].to(self.torch_device), state_dict_from_pretrained[key].to(self.torch_device) - ) - ) - - # check if `adapter_model.bin` is present - self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin"))) - - # check if `adapter_config.json` is present - self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json"))) - - # check if `pytorch_model.bin` is not present - self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin"))) - - # check if `config.json` is not present - self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) - - @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) - def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs): - self._test_save_pretrained(model_id, config_cls, config_kwargs) - - def _test_merge_layers(self, model_id, config_cls, config_kwargs): - model = AutoModelForCausalLM.from_pretrained(model_id) - config = config_cls( - base_model_name_or_path=model_id, - **config_kwargs, - ) - model = get_peft_model(model, config) - model = model.to(self.torch_device) - - if config.peft_type != "LORA": - with self.assertRaises(AttributeError): - model = model.merge_and_unload() - elif model.config.model_type == "gpt2": - with self.assertRaises(ValueError): - model = model.merge_and_unload() - else: - dummy_input = torch.LongTensor([[1, 2, 3, 2, 1]]).to(self.torch_device) - model.eval() - logits_lora = model(dummy_input)[0] - - model = model.merge_and_unload() - - logits_merged = model(dummy_input)[0] - - transformers_model = AutoModelForCausalLM.from_pretrained(model_id).to(self.torch_device) - - logits_transformers = transformers_model(dummy_input)[0] - - self.assertTrue(torch.allclose(logits_lora, logits_merged, atol=1e-3, rtol=1e-3)) - self.assertFalse(torch.allclose(logits_merged, logits_transformers, atol=1e-3, rtol=1e-3)) - - with tempfile.TemporaryDirectory() as tmp_dirname: - model.save_pretrained(tmp_dirname) - - model_from_pretrained = AutoModelForCausalLM.from_pretrained(tmp_dirname).to(self.torch_device) - - logits_merged_from_pretrained = model_from_pretrained(dummy_input)[0] - - self.assertTrue(torch.allclose(logits_merged, logits_merged_from_pretrained, atol=1e-3, rtol=1e-3)) - - @parameterized.expand( - PeftTestConfigManager.get_grid_parameters( - { - "model_ids": PEFT_DECODER_MODELS_TO_TEST, - "lora_kwargs": {"init_lora_weights": [False], "merge_weights": [False, True]}, - }, - ) - ) - def test_merge_layers(self, test_name, model_id, config_cls, config_kwargs): - self._test_merge_layers(model_id, config_cls, config_kwargs) - - def _test_generate(self, model_id, config_cls, config_kwargs): - model = AutoModelForCausalLM.from_pretrained(model_id) - config = config_cls( - base_model_name_or_path=model_id, - **config_kwargs, - ) - model = get_peft_model(model, config) - model = model.to(self.torch_device) - - input_ids = torch.LongTensor([[1, 1, 1], [2, 1, 2]]).to(self.torch_device) - attention_mask = torch.LongTensor([[1, 1, 1], [1, 0, 1]]).to(self.torch_device) - - # check if `generate` works - _ = model.generate(input_ids=input_ids, attention_mask=attention_mask) - - with self.assertRaises(TypeError): - # check if `generate` raises an error if no positional arguments are passed - _ = model.generate(input_ids, attention_mask=attention_mask) - - @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) - def test_generate(self, test_name, model_id, config_cls, config_kwargs): - self._test_generate(model_id, config_cls, config_kwargs) diff --git a/tests/testing_common.py b/tests/testing_common.py index 633bb87..cfe6cf2 100644 --- a/tests/testing_common.py +++ b/tests/testing_common.py @@ -12,13 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os +import tempfile from collections import OrderedDict +import torch + from peft import ( LoraConfig, + PeftModel, PrefixTuningConfig, PromptEncoderConfig, PromptTuningConfig, + get_peft_model, + get_peft_model_state_dict, + prepare_model_for_int8_training, ) @@ -35,20 +43,16 @@ CONFIG_TESTING_KWARGS = ( "target_modules": None, "lora_dropout": 0.05, "bias": "none", - "task_type": "CAUSAL_LM", }, { "num_virtual_tokens": 10, - "task_type": "CAUSAL_LM", }, { "num_virtual_tokens": 10, "encoder_hidden_size": 32, - "task_type": "CAUSAL_LM", }, { "num_virtual_tokens": 10, - "task_type": "CAUSAL_LM", }, ) @@ -92,6 +96,7 @@ class ClassInstantier(OrderedDict): """ generated_tests = [] model_list = grid_parameters["model_ids"] + task_type = grid_parameters["task_type"] if "task_type" in grid_parameters else None for model_id in model_list: for key, value in self.items(): @@ -101,9 +106,16 @@ class ClassInstantier(OrderedDict): for current_key, current_value in grid_parameters[f"{key}_kwargs"].items(): for kwarg in current_value: current_peft_config.update({current_key: kwarg}) - peft_configs.append(current_peft_config) + + if task_type is not None: + current_peft_config.update({"task_type": task_type}) + + peft_configs.append(current_peft_config.copy()) else: - peft_configs = [value[1].copy()] + current_peft_config = value[1].copy() + if task_type is not None: + current_peft_config.update({"task_type": task_type}) + peft_configs = [current_peft_config] for peft_config in peft_configs: generated_tests.append((f"test_{model_id}_{key}", model_id, value[0], peft_config)) @@ -115,3 +127,173 @@ class ClassInstantier(OrderedDict): PeftTestConfigManager = ClassInstantier(CLASSES_MAPPING) + + +class PeftCommonTester: + r""" + A large testing suite for testing common functionality of the PEFT models. + + Attributes: + torch_device (`torch.device`): + The device on which the tests will be run. + transformers_class (`transformers.PreTrainedModel`): + The transformers class that is being tested. + """ + torch_device = "cuda" if torch.cuda.is_available() else "cpu" + transformers_class = None + + def prepare_inputs_for_common(self): + raise NotImplementedError + + def _test_model_attr(self, model_id, config_cls, config_kwargs): + model = self.transformers_class.from_pretrained(model_id) + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + + self.assertTrue(hasattr(model, "save_pretrained")) + self.assertTrue(hasattr(model, "from_pretrained")) + self.assertTrue(hasattr(model, "push_to_hub")) + + def _test_prepare_for_training(self, model_id, config_cls, config_kwargs): + model = self.transformers_class.from_pretrained(model_id).to(self.torch_device) + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + + dummy_input = self.prepare_inputs_for_testing() + dummy_output = model.get_input_embeddings()(dummy_input["input_ids"]) + + self.assertTrue(not dummy_output.requires_grad) + + # load with `prepare_model_for_int8_training` + model = self.transformers_class.from_pretrained(model_id).to(self.torch_device) + model = prepare_model_for_int8_training(model) + + for param in model.parameters(): + self.assertTrue(not param.requires_grad) + + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + + # For backward compatibility + if hasattr(model, "enable_input_require_grads"): + model.enable_input_require_grads() + else: + + def make_inputs_require_grad(module, input, output): + output.requires_grad_(True) + + model.get_input_embeddings().register_forward_hook(make_inputs_require_grad) + + dummy_input = self.prepare_inputs_for_testing() + dummy_output = model.get_input_embeddings()(dummy_input["input_ids"]) + + self.assertTrue(dummy_output.requires_grad) + + def _test_save_pretrained(self, model_id, config_cls, config_kwargs): + model = self.transformers_class.from_pretrained(model_id) + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + model = model.to(self.torch_device) + + with tempfile.TemporaryDirectory() as tmp_dirname: + model.save_pretrained(tmp_dirname) + + model_from_pretrained = self.transformers_class.from_pretrained(model_id) + model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname) + + # check if the state dicts are equal + state_dict = get_peft_model_state_dict(model) + state_dict_from_pretrained = get_peft_model_state_dict(model_from_pretrained) + + # check if same keys + self.assertEqual(state_dict.keys(), state_dict_from_pretrained.keys()) + + # check if tensors equal + for key in state_dict.keys(): + self.assertTrue( + torch.allclose( + state_dict[key].to(self.torch_device), state_dict_from_pretrained[key].to(self.torch_device) + ) + ) + + # check if `adapter_model.bin` is present + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin"))) + + # check if `adapter_config.json` is present + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json"))) + + # check if `pytorch_model.bin` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin"))) + + # check if `config.json` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) + + def _test_merge_layers(self, model_id, config_cls, config_kwargs): + model = self.transformers_class.from_pretrained(model_id) + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + model = model.to(self.torch_device) + + if config.peft_type != "LORA": + with self.assertRaises(AttributeError): + model = model.merge_and_unload() + elif model.config.model_type == "gpt2": + with self.assertRaises(ValueError): + model = model.merge_and_unload() + else: + dummy_input = self.prepare_inputs_for_testing() + model.eval() + logits_lora = model(**dummy_input)[0] + + model = model.merge_and_unload() + + logits_merged = model(**dummy_input)[0] + + transformers_model = self.transformers_class.from_pretrained(model_id).to(self.torch_device) + + logits_transformers = transformers_model(**dummy_input)[0] + + self.assertTrue(torch.allclose(logits_lora, logits_merged, atol=1e-4, rtol=1e-4)) + self.assertFalse(torch.allclose(logits_merged, logits_transformers, atol=1e-10, rtol=1e-10)) + + with tempfile.TemporaryDirectory() as tmp_dirname: + model.save_pretrained(tmp_dirname) + + model_from_pretrained = self.transformers_class.from_pretrained(tmp_dirname).to(self.torch_device) + + logits_merged_from_pretrained = model_from_pretrained(**dummy_input)[0] + + self.assertTrue(torch.allclose(logits_merged, logits_merged_from_pretrained, atol=1e-4, rtol=1e-4)) + + def _test_generate(self, model_id, config_cls, config_kwargs): + model = self.transformers_class.from_pretrained(model_id) + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + model = model.to(self.torch_device) + + inputs = self.prepare_inputs_for_testing() + + # check if `generate` works + _ = model.generate(**inputs) + + with self.assertRaises(TypeError): + # check if `generate` raises an error if no positional arguments are passed + _ = model.generate(inputs["input_ids"])