mirror of
https://github.com/wassname/peft.git
synced 2026-09-09 11:28:32 +08:00
[tests] Adds more tests + fix failing tests (#238)
* adds more tests - refactor tests - add enc-dec tests - skips generate tests for non-lora adapters * rm unneeded file * fix tests * fix * more checks * fix issue
This commit is contained in:
Younes Belkada
+20
-5
@@ -270,11 +270,26 @@ class LoraModel(torch.nn.Module):
|
||||
# manually merge if not merged
|
||||
if not target.merged:
|
||||
# merge weights per: https://arxiv.org/pdf/2106.09685.pdf / page 4
|
||||
if target.r > 0:
|
||||
target.weight.data += (
|
||||
transpose(target.lora_B.weight @ target.lora_A.weight, target.fan_in_fan_out)
|
||||
* target.scaling
|
||||
).to(target.weight.dtype)
|
||||
if isinstance(target, Linear):
|
||||
if target.r > 0:
|
||||
target.weight.data += (
|
||||
transpose(target.lora_B.weight @ target.lora_A.weight, target.fan_in_fan_out)
|
||||
* target.scaling
|
||||
).to(target.weight.dtype)
|
||||
else:
|
||||
if target.r > 0:
|
||||
delta_w = (
|
||||
F.conv1d(
|
||||
target.lora_A.weight.data.unsqueeze(0),
|
||||
target.lora_B.weight.data,
|
||||
groups=sum(target.enable_lora),
|
||||
)
|
||||
.squeeze(0)
|
||||
.transpose(-2, -1)
|
||||
)
|
||||
target.weight.data += transpose(
|
||||
target.zero_pad(delta_w * target.scaling), not target.fan_in_fan_out
|
||||
).to(target.weight.dtype)
|
||||
target.merged = True
|
||||
|
||||
self._replace_module(parent, target_name, new_module, target)
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2023-present the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
from parameterized import parameterized
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
from .testing_common import PeftCommonTester, PeftTestConfigManager
|
||||
|
||||
|
||||
PEFT_DECODER_MODELS_TO_TEST = [
|
||||
"hf-internal-testing/tiny-random-OPTForCausalLM",
|
||||
"hf-internal-testing/tiny-random-GPTNeoXForCausalLM",
|
||||
"hf-internal-testing/tiny-random-GPT2LMHeadModel",
|
||||
"hf-internal-testing/tiny-random-BloomForCausalLM",
|
||||
"hf-internal-testing/tiny-random-gpt_neo",
|
||||
"hf-internal-testing/tiny-random-GPTJForCausalLM",
|
||||
]
|
||||
|
||||
FULL_GRID = {
|
||||
"model_ids": PEFT_DECODER_MODELS_TO_TEST,
|
||||
"task_type": "CAUSAL_LM",
|
||||
}
|
||||
|
||||
|
||||
class PeftDecoderModelTester(unittest.TestCase, PeftCommonTester):
|
||||
r"""
|
||||
Test if the PeftModel behaves as expected. This includes:
|
||||
- test if the model has the expected methods
|
||||
|
||||
We use parametrized.expand for debugging purposes to test each model individually.
|
||||
"""
|
||||
transformers_class = AutoModelForCausalLM
|
||||
|
||||
def prepare_inputs_for_testing(self):
|
||||
input_ids = torch.tensor([[1, 1, 1], [1, 2, 1]]).to(self.torch_device)
|
||||
attention_mask = torch.tensor([[1, 1, 1], [1, 0, 1]]).to(self.torch_device)
|
||||
|
||||
input_dict = {
|
||||
"input_ids": input_ids,
|
||||
"attention_mask": attention_mask,
|
||||
}
|
||||
|
||||
return input_dict
|
||||
|
||||
@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
|
||||
def test_attributes_parametrized(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_model_attr(model_id, config_cls, config_kwargs)
|
||||
|
||||
@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
|
||||
def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_prepare_for_training(model_id, config_cls, config_kwargs)
|
||||
|
||||
@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
|
||||
def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_save_pretrained(model_id, config_cls, config_kwargs)
|
||||
|
||||
@parameterized.expand(
|
||||
PeftTestConfigManager.get_grid_parameters(
|
||||
{
|
||||
"model_ids": PEFT_DECODER_MODELS_TO_TEST,
|
||||
"lora_kwargs": {"init_lora_weights": [False], "merge_weights": [False, True]},
|
||||
"task_type": "CAUSAL_LM",
|
||||
},
|
||||
)
|
||||
)
|
||||
def test_merge_layers(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_merge_layers(model_id, config_cls, config_kwargs)
|
||||
|
||||
@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
|
||||
def test_generate(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_generate(model_id, config_cls, config_kwargs)
|
||||
@@ -0,0 +1,88 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2023-present the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
from parameterized import parameterized
|
||||
from transformers import AutoModelForSeq2SeqLM
|
||||
|
||||
from .testing_common import PeftCommonTester, PeftTestConfigManager
|
||||
|
||||
|
||||
PEFT_ENCODER_DECODER_MODELS_TO_TEST = [
|
||||
"hf-internal-testing/tiny-random-T5ForConditionalGeneration",
|
||||
"hf-internal-testing/tiny-random-BartForConditionalGeneration",
|
||||
]
|
||||
|
||||
FULL_GRID = {"model_ids": PEFT_ENCODER_DECODER_MODELS_TO_TEST, "task_type": "SEQ_2_SEQ_LM"}
|
||||
|
||||
|
||||
def skip_non_lora_or_pt(test_list):
|
||||
r"""
|
||||
Skip tests that are not lora or prefix tuning
|
||||
"""
|
||||
return [test for test in test_list if ("lora" in test[0] or "prefix_tuning" in test[0])]
|
||||
|
||||
|
||||
class PeftEncoderDecoderModelTester(unittest.TestCase, PeftCommonTester):
|
||||
r"""
|
||||
Test if the PeftModel behaves as expected. This includes:
|
||||
- test if the model has the expected methods
|
||||
|
||||
We use parametrized.expand for debugging purposes to test each model individually.
|
||||
"""
|
||||
transformers_class = AutoModelForSeq2SeqLM
|
||||
|
||||
def prepare_inputs_for_testing(self):
|
||||
input_ids = torch.tensor([[1, 1, 1], [1, 2, 1]]).to(self.torch_device)
|
||||
decoder_input_ids = torch.tensor([[1, 1, 1], [1, 2, 1]]).to(self.torch_device)
|
||||
attention_mask = torch.tensor([[1, 1, 1], [1, 0, 1]]).to(self.torch_device)
|
||||
|
||||
input_dict = {
|
||||
"input_ids": input_ids,
|
||||
"decoder_input_ids": decoder_input_ids,
|
||||
"attention_mask": attention_mask,
|
||||
}
|
||||
|
||||
return input_dict
|
||||
|
||||
@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
|
||||
def test_attributes_parametrized(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_model_attr(model_id, config_cls, config_kwargs)
|
||||
|
||||
@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
|
||||
def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_prepare_for_training(model_id, config_cls, config_kwargs)
|
||||
|
||||
@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
|
||||
def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_save_pretrained(model_id, config_cls, config_kwargs)
|
||||
|
||||
@parameterized.expand(
|
||||
PeftTestConfigManager.get_grid_parameters(
|
||||
{
|
||||
"model_ids": PEFT_ENCODER_DECODER_MODELS_TO_TEST,
|
||||
"lora_kwargs": {"init_lora_weights": [False], "merge_weights": [True, False]},
|
||||
"task_type": "SEQ_2_SEQ_LM",
|
||||
},
|
||||
)
|
||||
)
|
||||
def test_merge_layers(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_merge_layers(model_id, config_cls, config_kwargs)
|
||||
|
||||
# skip non lora models - generate does not work for prefix tuning, prompt tuning
|
||||
@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID, filter_params_func=skip_non_lora_or_pt))
|
||||
def test_generate(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_generate(model_id, config_cls, config_kwargs)
|
||||
@@ -1,238 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2023-present the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
from parameterized import parameterized
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
from peft import (
|
||||
PeftModel,
|
||||
get_peft_model,
|
||||
get_peft_model_state_dict,
|
||||
prepare_model_for_int8_training,
|
||||
)
|
||||
|
||||
from .testing_common import PeftTestConfigManager
|
||||
|
||||
|
||||
PEFT_DECODER_MODELS_TO_TEST = [
|
||||
"hf-internal-testing/tiny-random-OPTForCausalLM",
|
||||
"hf-internal-testing/tiny-random-GPTNeoXForCausalLM",
|
||||
"hf-internal-testing/tiny-random-GPT2LMHeadModel",
|
||||
"hf-internal-testing/tiny-random-BloomForCausalLM",
|
||||
"hf-internal-testing/tiny-random-gpt_neo",
|
||||
"hf-internal-testing/tiny-random-GPTJForCausalLM",
|
||||
]
|
||||
|
||||
FULL_GRID = {
|
||||
"model_ids": PEFT_DECODER_MODELS_TO_TEST,
|
||||
}
|
||||
|
||||
|
||||
class PeftTestMixin:
|
||||
torch_device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
|
||||
class PeftModelTester(unittest.TestCase, PeftTestMixin):
|
||||
r"""
|
||||
Test if the PeftModel behaves as expected. This includes:
|
||||
- test if the model has the expected methods
|
||||
|
||||
We use parametrized.expand for debugging purposes to test each model individually.
|
||||
"""
|
||||
|
||||
def _test_model_attr(self, model_id, config_cls, config_kwargs):
|
||||
model = AutoModelForCausalLM.from_pretrained(model_id)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
|
||||
self.assertTrue(hasattr(model, "save_pretrained"))
|
||||
self.assertTrue(hasattr(model, "from_pretrained"))
|
||||
self.assertTrue(hasattr(model, "push_to_hub"))
|
||||
|
||||
@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
|
||||
def test_attributes_parametrized(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_model_attr(model_id, config_cls, config_kwargs)
|
||||
|
||||
def _test_prepare_for_training(self, model_id, config_cls, config_kwargs):
|
||||
model = AutoModelForCausalLM.from_pretrained(model_id).to(self.torch_device)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
|
||||
dummy_input = torch.LongTensor([[1, 1, 1]]).to(self.torch_device)
|
||||
dummy_output = model.get_input_embeddings()(dummy_input)
|
||||
|
||||
self.assertTrue(not dummy_output.requires_grad)
|
||||
|
||||
# load with `prepare_model_for_int8_training`
|
||||
model = AutoModelForCausalLM.from_pretrained(model_id).to(self.torch_device)
|
||||
model = prepare_model_for_int8_training(model)
|
||||
|
||||
for param in model.parameters():
|
||||
self.assertTrue(not param.requires_grad)
|
||||
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
|
||||
# For backward compatibility
|
||||
if hasattr(model, "enable_input_require_grads"):
|
||||
model.enable_input_require_grads()
|
||||
else:
|
||||
|
||||
def make_inputs_require_grad(module, input, output):
|
||||
output.requires_grad_(True)
|
||||
|
||||
model.get_input_embeddings().register_forward_hook(make_inputs_require_grad)
|
||||
|
||||
dummy_input = torch.LongTensor([[1, 1, 1]]).to(self.torch_device)
|
||||
dummy_output = model.get_input_embeddings()(dummy_input)
|
||||
|
||||
self.assertTrue(dummy_output.requires_grad)
|
||||
|
||||
@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
|
||||
def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_prepare_for_training(model_id, config_cls, config_kwargs)
|
||||
|
||||
def _test_save_pretrained(self, model_id, config_cls, config_kwargs):
|
||||
model = AutoModelForCausalLM.from_pretrained(model_id)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
model = model.to(self.torch_device)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dirname:
|
||||
model.save_pretrained(tmp_dirname)
|
||||
|
||||
model_from_pretrained = AutoModelForCausalLM.from_pretrained(model_id)
|
||||
model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname)
|
||||
|
||||
# check if the state dicts are equal
|
||||
state_dict = get_peft_model_state_dict(model)
|
||||
state_dict_from_pretrained = get_peft_model_state_dict(model_from_pretrained)
|
||||
|
||||
# check if same keys
|
||||
self.assertEqual(state_dict.keys(), state_dict_from_pretrained.keys())
|
||||
|
||||
# check if tensors equal
|
||||
for key in state_dict.keys():
|
||||
self.assertTrue(
|
||||
torch.allclose(
|
||||
state_dict[key].to(self.torch_device), state_dict_from_pretrained[key].to(self.torch_device)
|
||||
)
|
||||
)
|
||||
|
||||
# check if `adapter_model.bin` is present
|
||||
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin")))
|
||||
|
||||
# check if `adapter_config.json` is present
|
||||
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json")))
|
||||
|
||||
# check if `pytorch_model.bin` is not present
|
||||
self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin")))
|
||||
|
||||
# check if `config.json` is not present
|
||||
self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json")))
|
||||
|
||||
@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
|
||||
def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_save_pretrained(model_id, config_cls, config_kwargs)
|
||||
|
||||
def _test_merge_layers(self, model_id, config_cls, config_kwargs):
|
||||
model = AutoModelForCausalLM.from_pretrained(model_id)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
model = model.to(self.torch_device)
|
||||
|
||||
if config.peft_type != "LORA":
|
||||
with self.assertRaises(AttributeError):
|
||||
model = model.merge_and_unload()
|
||||
elif model.config.model_type == "gpt2":
|
||||
with self.assertRaises(ValueError):
|
||||
model = model.merge_and_unload()
|
||||
else:
|
||||
dummy_input = torch.LongTensor([[1, 2, 3, 2, 1]]).to(self.torch_device)
|
||||
model.eval()
|
||||
logits_lora = model(dummy_input)[0]
|
||||
|
||||
model = model.merge_and_unload()
|
||||
|
||||
logits_merged = model(dummy_input)[0]
|
||||
|
||||
transformers_model = AutoModelForCausalLM.from_pretrained(model_id).to(self.torch_device)
|
||||
|
||||
logits_transformers = transformers_model(dummy_input)[0]
|
||||
|
||||
self.assertTrue(torch.allclose(logits_lora, logits_merged, atol=1e-3, rtol=1e-3))
|
||||
self.assertFalse(torch.allclose(logits_merged, logits_transformers, atol=1e-3, rtol=1e-3))
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dirname:
|
||||
model.save_pretrained(tmp_dirname)
|
||||
|
||||
model_from_pretrained = AutoModelForCausalLM.from_pretrained(tmp_dirname).to(self.torch_device)
|
||||
|
||||
logits_merged_from_pretrained = model_from_pretrained(dummy_input)[0]
|
||||
|
||||
self.assertTrue(torch.allclose(logits_merged, logits_merged_from_pretrained, atol=1e-3, rtol=1e-3))
|
||||
|
||||
@parameterized.expand(
|
||||
PeftTestConfigManager.get_grid_parameters(
|
||||
{
|
||||
"model_ids": PEFT_DECODER_MODELS_TO_TEST,
|
||||
"lora_kwargs": {"init_lora_weights": [False], "merge_weights": [False, True]},
|
||||
},
|
||||
)
|
||||
)
|
||||
def test_merge_layers(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_merge_layers(model_id, config_cls, config_kwargs)
|
||||
|
||||
def _test_generate(self, model_id, config_cls, config_kwargs):
|
||||
model = AutoModelForCausalLM.from_pretrained(model_id)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
model = model.to(self.torch_device)
|
||||
|
||||
input_ids = torch.LongTensor([[1, 1, 1], [2, 1, 2]]).to(self.torch_device)
|
||||
attention_mask = torch.LongTensor([[1, 1, 1], [1, 0, 1]]).to(self.torch_device)
|
||||
|
||||
# check if `generate` works
|
||||
_ = model.generate(input_ids=input_ids, attention_mask=attention_mask)
|
||||
|
||||
with self.assertRaises(TypeError):
|
||||
# check if `generate` raises an error if no positional arguments are passed
|
||||
_ = model.generate(input_ids, attention_mask=attention_mask)
|
||||
|
||||
@parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID))
|
||||
def test_generate(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_generate(model_id, config_cls, config_kwargs)
|
||||
+188
-6
@@ -12,13 +12,21 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import os
|
||||
import tempfile
|
||||
from collections import OrderedDict
|
||||
|
||||
import torch
|
||||
|
||||
from peft import (
|
||||
LoraConfig,
|
||||
PeftModel,
|
||||
PrefixTuningConfig,
|
||||
PromptEncoderConfig,
|
||||
PromptTuningConfig,
|
||||
get_peft_model,
|
||||
get_peft_model_state_dict,
|
||||
prepare_model_for_int8_training,
|
||||
)
|
||||
|
||||
|
||||
@@ -35,20 +43,16 @@ CONFIG_TESTING_KWARGS = (
|
||||
"target_modules": None,
|
||||
"lora_dropout": 0.05,
|
||||
"bias": "none",
|
||||
"task_type": "CAUSAL_LM",
|
||||
},
|
||||
{
|
||||
"num_virtual_tokens": 10,
|
||||
"task_type": "CAUSAL_LM",
|
||||
},
|
||||
{
|
||||
"num_virtual_tokens": 10,
|
||||
"encoder_hidden_size": 32,
|
||||
"task_type": "CAUSAL_LM",
|
||||
},
|
||||
{
|
||||
"num_virtual_tokens": 10,
|
||||
"task_type": "CAUSAL_LM",
|
||||
},
|
||||
)
|
||||
|
||||
@@ -92,6 +96,7 @@ class ClassInstantier(OrderedDict):
|
||||
"""
|
||||
generated_tests = []
|
||||
model_list = grid_parameters["model_ids"]
|
||||
task_type = grid_parameters["task_type"] if "task_type" in grid_parameters else None
|
||||
|
||||
for model_id in model_list:
|
||||
for key, value in self.items():
|
||||
@@ -101,9 +106,16 @@ class ClassInstantier(OrderedDict):
|
||||
for current_key, current_value in grid_parameters[f"{key}_kwargs"].items():
|
||||
for kwarg in current_value:
|
||||
current_peft_config.update({current_key: kwarg})
|
||||
peft_configs.append(current_peft_config)
|
||||
|
||||
if task_type is not None:
|
||||
current_peft_config.update({"task_type": task_type})
|
||||
|
||||
peft_configs.append(current_peft_config.copy())
|
||||
else:
|
||||
peft_configs = [value[1].copy()]
|
||||
current_peft_config = value[1].copy()
|
||||
if task_type is not None:
|
||||
current_peft_config.update({"task_type": task_type})
|
||||
peft_configs = [current_peft_config]
|
||||
|
||||
for peft_config in peft_configs:
|
||||
generated_tests.append((f"test_{model_id}_{key}", model_id, value[0], peft_config))
|
||||
@@ -115,3 +127,173 @@ class ClassInstantier(OrderedDict):
|
||||
|
||||
|
||||
PeftTestConfigManager = ClassInstantier(CLASSES_MAPPING)
|
||||
|
||||
|
||||
class PeftCommonTester:
|
||||
r"""
|
||||
A large testing suite for testing common functionality of the PEFT models.
|
||||
|
||||
Attributes:
|
||||
torch_device (`torch.device`):
|
||||
The device on which the tests will be run.
|
||||
transformers_class (`transformers.PreTrainedModel`):
|
||||
The transformers class that is being tested.
|
||||
"""
|
||||
torch_device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
transformers_class = None
|
||||
|
||||
def prepare_inputs_for_common(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def _test_model_attr(self, model_id, config_cls, config_kwargs):
|
||||
model = self.transformers_class.from_pretrained(model_id)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
|
||||
self.assertTrue(hasattr(model, "save_pretrained"))
|
||||
self.assertTrue(hasattr(model, "from_pretrained"))
|
||||
self.assertTrue(hasattr(model, "push_to_hub"))
|
||||
|
||||
def _test_prepare_for_training(self, model_id, config_cls, config_kwargs):
|
||||
model = self.transformers_class.from_pretrained(model_id).to(self.torch_device)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
|
||||
dummy_input = self.prepare_inputs_for_testing()
|
||||
dummy_output = model.get_input_embeddings()(dummy_input["input_ids"])
|
||||
|
||||
self.assertTrue(not dummy_output.requires_grad)
|
||||
|
||||
# load with `prepare_model_for_int8_training`
|
||||
model = self.transformers_class.from_pretrained(model_id).to(self.torch_device)
|
||||
model = prepare_model_for_int8_training(model)
|
||||
|
||||
for param in model.parameters():
|
||||
self.assertTrue(not param.requires_grad)
|
||||
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
|
||||
# For backward compatibility
|
||||
if hasattr(model, "enable_input_require_grads"):
|
||||
model.enable_input_require_grads()
|
||||
else:
|
||||
|
||||
def make_inputs_require_grad(module, input, output):
|
||||
output.requires_grad_(True)
|
||||
|
||||
model.get_input_embeddings().register_forward_hook(make_inputs_require_grad)
|
||||
|
||||
dummy_input = self.prepare_inputs_for_testing()
|
||||
dummy_output = model.get_input_embeddings()(dummy_input["input_ids"])
|
||||
|
||||
self.assertTrue(dummy_output.requires_grad)
|
||||
|
||||
def _test_save_pretrained(self, model_id, config_cls, config_kwargs):
|
||||
model = self.transformers_class.from_pretrained(model_id)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
model = model.to(self.torch_device)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dirname:
|
||||
model.save_pretrained(tmp_dirname)
|
||||
|
||||
model_from_pretrained = self.transformers_class.from_pretrained(model_id)
|
||||
model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname)
|
||||
|
||||
# check if the state dicts are equal
|
||||
state_dict = get_peft_model_state_dict(model)
|
||||
state_dict_from_pretrained = get_peft_model_state_dict(model_from_pretrained)
|
||||
|
||||
# check if same keys
|
||||
self.assertEqual(state_dict.keys(), state_dict_from_pretrained.keys())
|
||||
|
||||
# check if tensors equal
|
||||
for key in state_dict.keys():
|
||||
self.assertTrue(
|
||||
torch.allclose(
|
||||
state_dict[key].to(self.torch_device), state_dict_from_pretrained[key].to(self.torch_device)
|
||||
)
|
||||
)
|
||||
|
||||
# check if `adapter_model.bin` is present
|
||||
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin")))
|
||||
|
||||
# check if `adapter_config.json` is present
|
||||
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json")))
|
||||
|
||||
# check if `pytorch_model.bin` is not present
|
||||
self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin")))
|
||||
|
||||
# check if `config.json` is not present
|
||||
self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json")))
|
||||
|
||||
def _test_merge_layers(self, model_id, config_cls, config_kwargs):
|
||||
model = self.transformers_class.from_pretrained(model_id)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
model = model.to(self.torch_device)
|
||||
|
||||
if config.peft_type != "LORA":
|
||||
with self.assertRaises(AttributeError):
|
||||
model = model.merge_and_unload()
|
||||
elif model.config.model_type == "gpt2":
|
||||
with self.assertRaises(ValueError):
|
||||
model = model.merge_and_unload()
|
||||
else:
|
||||
dummy_input = self.prepare_inputs_for_testing()
|
||||
model.eval()
|
||||
logits_lora = model(**dummy_input)[0]
|
||||
|
||||
model = model.merge_and_unload()
|
||||
|
||||
logits_merged = model(**dummy_input)[0]
|
||||
|
||||
transformers_model = self.transformers_class.from_pretrained(model_id).to(self.torch_device)
|
||||
|
||||
logits_transformers = transformers_model(**dummy_input)[0]
|
||||
|
||||
self.assertTrue(torch.allclose(logits_lora, logits_merged, atol=1e-4, rtol=1e-4))
|
||||
self.assertFalse(torch.allclose(logits_merged, logits_transformers, atol=1e-10, rtol=1e-10))
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dirname:
|
||||
model.save_pretrained(tmp_dirname)
|
||||
|
||||
model_from_pretrained = self.transformers_class.from_pretrained(tmp_dirname).to(self.torch_device)
|
||||
|
||||
logits_merged_from_pretrained = model_from_pretrained(**dummy_input)[0]
|
||||
|
||||
self.assertTrue(torch.allclose(logits_merged, logits_merged_from_pretrained, atol=1e-4, rtol=1e-4))
|
||||
|
||||
def _test_generate(self, model_id, config_cls, config_kwargs):
|
||||
model = self.transformers_class.from_pretrained(model_id)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
model = model.to(self.torch_device)
|
||||
|
||||
inputs = self.prepare_inputs_for_testing()
|
||||
|
||||
# check if `generate` works
|
||||
_ = model.generate(**inputs)
|
||||
|
||||
with self.assertRaises(TypeError):
|
||||
# check if `generate` raises an error if no positional arguments are passed
|
||||
_ = model.generate(inputs["input_ids"])
|
||||
|
||||
Reference in New Issue
Block a user