mirror of
https://github.com/wassname/peft.git
synced 2026-09-09 11:28:32 +08:00
[CI] Add ci tests (#203)
* add ci tests * fix some tests * fix tests * rename * fix * update tests * try * temp hotfix * refactor tests * Update .github/workflows/tests.yml * fix test
This commit is contained in:
Younes Belkada
@@ -0,0 +1,46 @@
|
||||
name: tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
|
||||
check_code_quality:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.8"
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install .[dev]
|
||||
- name: Check quality
|
||||
run: |
|
||||
make quality
|
||||
|
||||
tests:
|
||||
needs: check_code_quality
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: [3.8, 3.9, 3.10]
|
||||
os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
# cpu version of pytorch
|
||||
pip install .[test]
|
||||
- name: Test with pytest
|
||||
run: |
|
||||
make test
|
||||
@@ -15,4 +15,6 @@ style:
|
||||
black $(check_dirs)
|
||||
ruff $(check_dirs) --fix
|
||||
doc-builder style src tests --max_len 119
|
||||
|
||||
|
||||
test:
|
||||
pytest tests/
|
||||
+113
-130
@@ -17,157 +17,140 @@ import tempfile
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
from parameterized import parameterized
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
from peft import (
|
||||
LoraConfig,
|
||||
PeftModel,
|
||||
PrefixTuningConfig,
|
||||
PromptEncoderConfig,
|
||||
PromptTuningConfig,
|
||||
get_peft_model,
|
||||
get_peft_model_state_dict,
|
||||
prepare_model_for_training,
|
||||
prepare_model_for_int8_training,
|
||||
)
|
||||
|
||||
from .testing_common import PeftTestConfigManager
|
||||
|
||||
|
||||
# This has to be in the order: model_id, lora_kwargs, prefix_tuning_kwargs, prompt_encoder_kwargs, prompt_tuning_kwargs
|
||||
PEFT_MODELS_TO_TEST = [
|
||||
("hf-internal-testing/tiny-random-OPTForCausalLM", {"target_modules": ["q_proj", "v_proj"]}, {}, {}, {}),
|
||||
]
|
||||
|
||||
|
||||
class PeftTestMixin:
|
||||
checkpoints_to_test = [
|
||||
"hf-internal-testing/tiny-random-OPTForCausalLM",
|
||||
]
|
||||
config_classes = (
|
||||
LoraConfig,
|
||||
PrefixTuningConfig,
|
||||
PromptEncoderConfig,
|
||||
PromptTuningConfig,
|
||||
)
|
||||
config_kwargs = (
|
||||
{
|
||||
"r": 8,
|
||||
"lora_alpha": 32,
|
||||
"target_modules": ["q_proj", "v_proj"],
|
||||
"lora_dropout": 0.05,
|
||||
"bias": "none",
|
||||
"task_type": "CAUSAL_LM",
|
||||
},
|
||||
{
|
||||
"num_virtual_tokens": 10,
|
||||
"task_type": "CAUSAL_LM",
|
||||
},
|
||||
{
|
||||
"num_virtual_tokens": 10,
|
||||
"encoder_hidden_size": 32,
|
||||
"task_type": "CAUSAL_LM",
|
||||
},
|
||||
{
|
||||
"num_virtual_tokens": 10,
|
||||
"task_type": "CAUSAL_LM",
|
||||
},
|
||||
)
|
||||
torch_device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
|
||||
class PeftModelTester(unittest.TestCase, PeftTestMixin):
|
||||
r"""
|
||||
Test if the PeftModel behaves as expected. This includes:
|
||||
- test if the model has the expected methods
|
||||
|
||||
We use parametrized.expand for debugging purposes to test each model individually.
|
||||
"""
|
||||
|
||||
def test_attributes_model(self):
|
||||
for model_id in self.checkpoints_to_test:
|
||||
for i, config_cls in enumerate(self.config_classes):
|
||||
model = AutoModelForCausalLM.from_pretrained(model_id)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**self.config_kwargs[i],
|
||||
@parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_MODELS_TO_TEST))
|
||||
def test_attributes_parametrized(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_model_attr(model_id, config_cls, config_kwargs)
|
||||
|
||||
def _test_model_attr(self, model_id, config_cls, config_kwargs):
|
||||
model = AutoModelForCausalLM.from_pretrained(model_id)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
|
||||
self.assertTrue(hasattr(model, "save_pretrained"))
|
||||
self.assertTrue(hasattr(model, "from_pretrained"))
|
||||
self.assertTrue(hasattr(model, "push_to_hub"))
|
||||
|
||||
def _test_prepare_for_training(self, model_id, config_cls, config_kwargs):
|
||||
model = AutoModelForCausalLM.from_pretrained(model_id).to(self.torch_device)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
|
||||
dummy_input = torch.LongTensor([[1, 1, 1]]).to(self.torch_device)
|
||||
dummy_output = model.get_input_embeddings()(dummy_input)
|
||||
|
||||
self.assertTrue(not dummy_output.requires_grad)
|
||||
|
||||
# load with `prepare_model_for_int8_training`
|
||||
model = AutoModelForCausalLM.from_pretrained(model_id).to(self.torch_device)
|
||||
model = prepare_model_for_int8_training(model)
|
||||
|
||||
for param in model.parameters():
|
||||
self.assertTrue(not param.requires_grad)
|
||||
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
|
||||
# For backward compatibility
|
||||
if hasattr(model, "enable_input_require_grads"):
|
||||
model.enable_input_require_grads()
|
||||
else:
|
||||
|
||||
def make_inputs_require_grad(module, input, output):
|
||||
output.requires_grad_(True)
|
||||
|
||||
model.get_input_embeddings().register_forward_hook(make_inputs_require_grad)
|
||||
|
||||
dummy_input = torch.LongTensor([[1, 1, 1]]).to(self.torch_device)
|
||||
dummy_output = model.get_input_embeddings()(dummy_input)
|
||||
|
||||
self.assertTrue(dummy_output.requires_grad)
|
||||
|
||||
@parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_MODELS_TO_TEST))
|
||||
def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_prepare_for_training(model_id, config_cls, config_kwargs)
|
||||
|
||||
def _test_save_pretrained(self, model_id, config_cls, config_kwargs):
|
||||
model = AutoModelForCausalLM.from_pretrained(model_id)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**config_kwargs,
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
model = model.to(self.torch_device)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dirname:
|
||||
model.save_pretrained(tmp_dirname)
|
||||
|
||||
model_from_pretrained = AutoModelForCausalLM.from_pretrained(model_id)
|
||||
model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname)
|
||||
|
||||
# check if the state dicts are equal
|
||||
state_dict = get_peft_model_state_dict(model)
|
||||
state_dict_from_pretrained = get_peft_model_state_dict(model_from_pretrained)
|
||||
|
||||
# check if same keys
|
||||
self.assertEqual(state_dict.keys(), state_dict_from_pretrained.keys())
|
||||
|
||||
# check if tensors equal
|
||||
for key in state_dict.keys():
|
||||
self.assertTrue(
|
||||
torch.allclose(
|
||||
state_dict[key].to(self.torch_device), state_dict_from_pretrained[key].to(self.torch_device)
|
||||
)
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
|
||||
self.assertTrue(hasattr(model, "save_pretrained"))
|
||||
self.assertTrue(hasattr(model, "from_pretrained"))
|
||||
self.assertTrue(hasattr(model, "push_to_hub"))
|
||||
# check if `adapter_model.bin` is present
|
||||
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin")))
|
||||
|
||||
def test_prepare_for_training(self):
|
||||
r"""
|
||||
A test that checks if `prepare_for_training` behaves as expected
|
||||
"""
|
||||
for model_id in self.checkpoints_to_test:
|
||||
for i, config_cls in enumerate(self.config_classes):
|
||||
model = AutoModelForCausalLM.from_pretrained(model_id)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**self.config_kwargs[i],
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
# check if `adapter_config.json` is present
|
||||
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json")))
|
||||
|
||||
dummy_input = torch.LongTensor([[1, 1, 1]])
|
||||
dummy_output = model.get_input_embeddings()(dummy_input)
|
||||
# check if `pytorch_model.bin` is not present
|
||||
self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin")))
|
||||
|
||||
self.assertTrue(not dummy_output.requires_grad)
|
||||
# check if `config.json` is not present
|
||||
self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json")))
|
||||
|
||||
# load with `prepare_model_for_training`
|
||||
model = AutoModelForCausalLM.from_pretrained(model_id)
|
||||
model = prepare_model_for_training(model)
|
||||
|
||||
for param in model.parameters():
|
||||
self.assertTrue(not param.requires_grad)
|
||||
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**self.config_kwargs[i],
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
|
||||
dummy_input = torch.LongTensor([[1, 1, 1]])
|
||||
dummy_output = model.get_input_embeddings()(dummy_input)
|
||||
|
||||
self.assertTrue(dummy_output.requires_grad)
|
||||
|
||||
def test_save_pretrained(self):
|
||||
r"""
|
||||
A test to check if `save_pretrained` behaves as expected. This function should only save the state dict of the
|
||||
adapter model and not the state dict of the base model. Hence inside each saved directory you should have:
|
||||
|
||||
- README.md (that contains an entry `base_model`)
|
||||
- adapter_config.json
|
||||
- adapter_model.bin
|
||||
|
||||
"""
|
||||
for model_id in self.checkpoints_to_test:
|
||||
for i, config_cls in enumerate(self.config_classes):
|
||||
model = AutoModelForCausalLM.from_pretrained(model_id)
|
||||
config = config_cls(
|
||||
base_model_name_or_path=model_id,
|
||||
**self.config_kwargs[i],
|
||||
)
|
||||
model = get_peft_model(model, config)
|
||||
model.to(model.device)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dirname:
|
||||
model.save_pretrained(tmp_dirname)
|
||||
|
||||
model_from_pretrained = AutoModelForCausalLM.from_pretrained(model_id)
|
||||
model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname)
|
||||
model_from_pretrained.to(model.device)
|
||||
|
||||
# check if the state dicts are equal
|
||||
state_dict = get_peft_model_state_dict(model)
|
||||
state_dict_from_pretrained = get_peft_model_state_dict(model_from_pretrained)
|
||||
|
||||
# check if same keys
|
||||
self.assertEqual(state_dict.keys(), state_dict_from_pretrained.keys())
|
||||
|
||||
# check if tensors equal
|
||||
for key in state_dict.keys():
|
||||
self.assertTrue(torch.allclose(state_dict[key], state_dict_from_pretrained[key]))
|
||||
|
||||
# check if `adapter_model.bin` is present
|
||||
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin")))
|
||||
|
||||
# check if `adapter_config.json` is present
|
||||
self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json")))
|
||||
|
||||
# check if `pytorch_model.bin` is not present
|
||||
self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin")))
|
||||
|
||||
# check if `config.json` is not present
|
||||
self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json")))
|
||||
@parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_MODELS_TO_TEST))
|
||||
def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs):
|
||||
self._test_save_pretrained(model_id, config_cls, config_kwargs)
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2023-present the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from collections import OrderedDict
|
||||
|
||||
from peft import (
|
||||
LoraConfig,
|
||||
PrefixTuningConfig,
|
||||
PromptEncoderConfig,
|
||||
PromptTuningConfig,
|
||||
)
|
||||
|
||||
|
||||
CONFIG_CLASSES = (
|
||||
LoraConfig,
|
||||
PrefixTuningConfig,
|
||||
PromptEncoderConfig,
|
||||
PromptTuningConfig,
|
||||
)
|
||||
CONFIG_TESTING_KWARGS = (
|
||||
{
|
||||
"r": 8,
|
||||
"lora_alpha": 32,
|
||||
"target_modules": None,
|
||||
"lora_dropout": 0.05,
|
||||
"bias": "none",
|
||||
"task_type": "CAUSAL_LM",
|
||||
},
|
||||
{
|
||||
"num_virtual_tokens": 10,
|
||||
"task_type": "CAUSAL_LM",
|
||||
},
|
||||
{
|
||||
"num_virtual_tokens": 10,
|
||||
"encoder_hidden_size": 32,
|
||||
"task_type": "CAUSAL_LM",
|
||||
},
|
||||
{
|
||||
"num_virtual_tokens": 10,
|
||||
"task_type": "CAUSAL_LM",
|
||||
},
|
||||
)
|
||||
|
||||
CLASSES_MAPPING = {
|
||||
"lora": (LoraConfig, CONFIG_TESTING_KWARGS[0]),
|
||||
"prefix_tuning": (PrefixTuningConfig, CONFIG_TESTING_KWARGS[1]),
|
||||
"prompt_encoder": (PromptEncoderConfig, CONFIG_TESTING_KWARGS[2]),
|
||||
"prompt_tuning": (PromptTuningConfig, CONFIG_TESTING_KWARGS[3]),
|
||||
}
|
||||
|
||||
|
||||
# Adapted from https://github.com/huggingface/transformers/blob/48327c57182fdade7f7797d1eaad2d166de5c55b/src/transformers/activations.py#LL166C7-L166C22
|
||||
class ClassInstantier(OrderedDict):
|
||||
def __getitem__(self, key, *args, **kwargs):
|
||||
# check if any of the kwargs is inside the config class kwargs
|
||||
if any([kwarg in self[key][1] for kwarg in kwargs]):
|
||||
new_config_kwargs = self[key][1].copy()
|
||||
new_config_kwargs.update(kwargs)
|
||||
return (self[key][0], new_config_kwargs)
|
||||
|
||||
return super().__getitem__(key, *args, **kwargs)
|
||||
|
||||
def get_grid_parameters(self, model_list):
|
||||
r"""
|
||||
Returns a list of all possible combinations of the parameters in the config classes.
|
||||
"""
|
||||
grid_parameters = []
|
||||
for model_tuple in model_list:
|
||||
model_id, lora_kwargs, prefix_tuning_kwargs, prompt_encoder_kwargs, prompt_tuning_kwargs = model_tuple
|
||||
for key, value in self.items():
|
||||
if key == "lora":
|
||||
# update value[1] if necessary
|
||||
if lora_kwargs is not None:
|
||||
value[1].update(lora_kwargs)
|
||||
elif key == "prefix_tuning":
|
||||
# update value[1] if necessary
|
||||
if prefix_tuning_kwargs is not None:
|
||||
value[1].update(prefix_tuning_kwargs)
|
||||
elif key == "prompt_encoder":
|
||||
# update value[1] if necessary
|
||||
if prompt_encoder_kwargs is not None:
|
||||
value[1].update(prompt_encoder_kwargs)
|
||||
else:
|
||||
# update value[1] if necessary
|
||||
if prompt_tuning_kwargs is not None:
|
||||
value[1].update(prompt_tuning_kwargs)
|
||||
grid_parameters.append((f"test_{model_id}_{key}", model_id, value[0], value[1]))
|
||||
|
||||
return grid_parameters
|
||||
|
||||
|
||||
PeftTestConfigManager = ClassInstantier(CLASSES_MAPPING)
|
||||
@@ -0,0 +1,49 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2023-present the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def require_torch_gpu(test_case):
|
||||
"""
|
||||
Decorator marking a test that requires a GPU. Will be skipped when no GPU is available.
|
||||
"""
|
||||
if not torch.cuda.is_available():
|
||||
return unittest.skip("test requires GPU")(test_case)
|
||||
else:
|
||||
return test_case
|
||||
|
||||
|
||||
def require_torch_multi_gpu(test_case):
|
||||
"""
|
||||
Decorator marking a test that requires multiple GPUs. Will be skipped when less than 2 GPUs are available.
|
||||
"""
|
||||
if not torch.cuda.is_available() or torch.cuda.device_count() < 2:
|
||||
return unittest.skip("test requires multiple GPUs")(test_case)
|
||||
else:
|
||||
return test_case
|
||||
|
||||
|
||||
def require_bitsandbytes(test_case):
|
||||
"""
|
||||
Decorator marking a test that requires the bitsandbytes library. Will be skipped when the library is not installed.
|
||||
"""
|
||||
try:
|
||||
import bitsandbytes # noqa: F401
|
||||
except ImportError:
|
||||
return unittest.skip("test requires bitsandbytes")(test_case)
|
||||
else:
|
||||
return test_case
|
||||
Reference in New Issue
Block a user