diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..8bb2491 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,46 @@ +name: tests + +on: + push: + branches: [ main ] + pull_request: + +jobs: + + check_code_quality: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.8" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install .[dev] + - name: Check quality + run: | + make quality + + tests: + needs: check_code_quality + strategy: + matrix: + python-version: [3.8, 3.9, 3.10] + os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + # cpu version of pytorch + pip install .[test] + - name: Test with pytest + run: | + make test \ No newline at end of file diff --git a/Makefile b/Makefile index ff8ed42..61549db 100644 --- a/Makefile +++ b/Makefile @@ -15,4 +15,6 @@ style: black $(check_dirs) ruff $(check_dirs) --fix doc-builder style src tests --max_len 119 - \ No newline at end of file + +test: + pytest tests/ \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_peft_model.py b/tests/test_peft_model.py index 0e85c7c..2ca4895 100644 --- a/tests/test_peft_model.py +++ b/tests/test_peft_model.py @@ -17,157 +17,140 @@ import tempfile import unittest import torch +from parameterized import parameterized from transformers import AutoModelForCausalLM from peft import ( - LoraConfig, PeftModel, - PrefixTuningConfig, - PromptEncoderConfig, - PromptTuningConfig, get_peft_model, get_peft_model_state_dict, - prepare_model_for_training, + prepare_model_for_int8_training, ) +from .testing_common import PeftTestConfigManager + + +# This has to be in the order: model_id, lora_kwargs, prefix_tuning_kwargs, prompt_encoder_kwargs, prompt_tuning_kwargs +PEFT_MODELS_TO_TEST = [ + ("hf-internal-testing/tiny-random-OPTForCausalLM", {"target_modules": ["q_proj", "v_proj"]}, {}, {}, {}), +] + class PeftTestMixin: - checkpoints_to_test = [ - "hf-internal-testing/tiny-random-OPTForCausalLM", - ] - config_classes = ( - LoraConfig, - PrefixTuningConfig, - PromptEncoderConfig, - PromptTuningConfig, - ) - config_kwargs = ( - { - "r": 8, - "lora_alpha": 32, - "target_modules": ["q_proj", "v_proj"], - "lora_dropout": 0.05, - "bias": "none", - "task_type": "CAUSAL_LM", - }, - { - "num_virtual_tokens": 10, - "task_type": "CAUSAL_LM", - }, - { - "num_virtual_tokens": 10, - "encoder_hidden_size": 32, - "task_type": "CAUSAL_LM", - }, - { - "num_virtual_tokens": 10, - "task_type": "CAUSAL_LM", - }, - ) + torch_device = "cuda" if torch.cuda.is_available() else "cpu" class PeftModelTester(unittest.TestCase, PeftTestMixin): r""" Test if the PeftModel behaves as expected. This includes: - test if the model has the expected methods + + We use parametrized.expand for debugging purposes to test each model individually. """ - def test_attributes_model(self): - for model_id in self.checkpoints_to_test: - for i, config_cls in enumerate(self.config_classes): - model = AutoModelForCausalLM.from_pretrained(model_id) - config = config_cls( - base_model_name_or_path=model_id, - **self.config_kwargs[i], + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_MODELS_TO_TEST)) + def test_attributes_parametrized(self, test_name, model_id, config_cls, config_kwargs): + self._test_model_attr(model_id, config_cls, config_kwargs) + + def _test_model_attr(self, model_id, config_cls, config_kwargs): + model = AutoModelForCausalLM.from_pretrained(model_id) + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + + self.assertTrue(hasattr(model, "save_pretrained")) + self.assertTrue(hasattr(model, "from_pretrained")) + self.assertTrue(hasattr(model, "push_to_hub")) + + def _test_prepare_for_training(self, model_id, config_cls, config_kwargs): + model = AutoModelForCausalLM.from_pretrained(model_id).to(self.torch_device) + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + + dummy_input = torch.LongTensor([[1, 1, 1]]).to(self.torch_device) + dummy_output = model.get_input_embeddings()(dummy_input) + + self.assertTrue(not dummy_output.requires_grad) + + # load with `prepare_model_for_int8_training` + model = AutoModelForCausalLM.from_pretrained(model_id).to(self.torch_device) + model = prepare_model_for_int8_training(model) + + for param in model.parameters(): + self.assertTrue(not param.requires_grad) + + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + + # For backward compatibility + if hasattr(model, "enable_input_require_grads"): + model.enable_input_require_grads() + else: + + def make_inputs_require_grad(module, input, output): + output.requires_grad_(True) + + model.get_input_embeddings().register_forward_hook(make_inputs_require_grad) + + dummy_input = torch.LongTensor([[1, 1, 1]]).to(self.torch_device) + dummy_output = model.get_input_embeddings()(dummy_input) + + self.assertTrue(dummy_output.requires_grad) + + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_MODELS_TO_TEST)) + def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls, config_kwargs): + self._test_prepare_for_training(model_id, config_cls, config_kwargs) + + def _test_save_pretrained(self, model_id, config_cls, config_kwargs): + model = AutoModelForCausalLM.from_pretrained(model_id) + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + model = model.to(self.torch_device) + + with tempfile.TemporaryDirectory() as tmp_dirname: + model.save_pretrained(tmp_dirname) + + model_from_pretrained = AutoModelForCausalLM.from_pretrained(model_id) + model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname) + + # check if the state dicts are equal + state_dict = get_peft_model_state_dict(model) + state_dict_from_pretrained = get_peft_model_state_dict(model_from_pretrained) + + # check if same keys + self.assertEqual(state_dict.keys(), state_dict_from_pretrained.keys()) + + # check if tensors equal + for key in state_dict.keys(): + self.assertTrue( + torch.allclose( + state_dict[key].to(self.torch_device), state_dict_from_pretrained[key].to(self.torch_device) + ) ) - model = get_peft_model(model, config) - self.assertTrue(hasattr(model, "save_pretrained")) - self.assertTrue(hasattr(model, "from_pretrained")) - self.assertTrue(hasattr(model, "push_to_hub")) + # check if `adapter_model.bin` is present + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin"))) - def test_prepare_for_training(self): - r""" - A test that checks if `prepare_for_training` behaves as expected - """ - for model_id in self.checkpoints_to_test: - for i, config_cls in enumerate(self.config_classes): - model = AutoModelForCausalLM.from_pretrained(model_id) - config = config_cls( - base_model_name_or_path=model_id, - **self.config_kwargs[i], - ) - model = get_peft_model(model, config) + # check if `adapter_config.json` is present + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json"))) - dummy_input = torch.LongTensor([[1, 1, 1]]) - dummy_output = model.get_input_embeddings()(dummy_input) + # check if `pytorch_model.bin` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin"))) - self.assertTrue(not dummy_output.requires_grad) + # check if `config.json` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) - # load with `prepare_model_for_training` - model = AutoModelForCausalLM.from_pretrained(model_id) - model = prepare_model_for_training(model) - - for param in model.parameters(): - self.assertTrue(not param.requires_grad) - - config = config_cls( - base_model_name_or_path=model_id, - **self.config_kwargs[i], - ) - model = get_peft_model(model, config) - - dummy_input = torch.LongTensor([[1, 1, 1]]) - dummy_output = model.get_input_embeddings()(dummy_input) - - self.assertTrue(dummy_output.requires_grad) - - def test_save_pretrained(self): - r""" - A test to check if `save_pretrained` behaves as expected. This function should only save the state dict of the - adapter model and not the state dict of the base model. Hence inside each saved directory you should have: - - - README.md (that contains an entry `base_model`) - - adapter_config.json - - adapter_model.bin - - """ - for model_id in self.checkpoints_to_test: - for i, config_cls in enumerate(self.config_classes): - model = AutoModelForCausalLM.from_pretrained(model_id) - config = config_cls( - base_model_name_or_path=model_id, - **self.config_kwargs[i], - ) - model = get_peft_model(model, config) - model.to(model.device) - - with tempfile.TemporaryDirectory() as tmp_dirname: - model.save_pretrained(tmp_dirname) - - model_from_pretrained = AutoModelForCausalLM.from_pretrained(model_id) - model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname) - model_from_pretrained.to(model.device) - - # check if the state dicts are equal - state_dict = get_peft_model_state_dict(model) - state_dict_from_pretrained = get_peft_model_state_dict(model_from_pretrained) - - # check if same keys - self.assertEqual(state_dict.keys(), state_dict_from_pretrained.keys()) - - # check if tensors equal - for key in state_dict.keys(): - self.assertTrue(torch.allclose(state_dict[key], state_dict_from_pretrained[key])) - - # check if `adapter_model.bin` is present - self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin"))) - - # check if `adapter_config.json` is present - self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json"))) - - # check if `pytorch_model.bin` is not present - self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin"))) - - # check if `config.json` is not present - self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_MODELS_TO_TEST)) + def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs): + self._test_save_pretrained(model_id, config_cls, config_kwargs) diff --git a/tests/testing_common.py b/tests/testing_common.py new file mode 100644 index 0000000..dfdf1d8 --- /dev/null +++ b/tests/testing_common.py @@ -0,0 +1,103 @@ +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import OrderedDict + +from peft import ( + LoraConfig, + PrefixTuningConfig, + PromptEncoderConfig, + PromptTuningConfig, +) + + +CONFIG_CLASSES = ( + LoraConfig, + PrefixTuningConfig, + PromptEncoderConfig, + PromptTuningConfig, +) +CONFIG_TESTING_KWARGS = ( + { + "r": 8, + "lora_alpha": 32, + "target_modules": None, + "lora_dropout": 0.05, + "bias": "none", + "task_type": "CAUSAL_LM", + }, + { + "num_virtual_tokens": 10, + "task_type": "CAUSAL_LM", + }, + { + "num_virtual_tokens": 10, + "encoder_hidden_size": 32, + "task_type": "CAUSAL_LM", + }, + { + "num_virtual_tokens": 10, + "task_type": "CAUSAL_LM", + }, +) + +CLASSES_MAPPING = { + "lora": (LoraConfig, CONFIG_TESTING_KWARGS[0]), + "prefix_tuning": (PrefixTuningConfig, CONFIG_TESTING_KWARGS[1]), + "prompt_encoder": (PromptEncoderConfig, CONFIG_TESTING_KWARGS[2]), + "prompt_tuning": (PromptTuningConfig, CONFIG_TESTING_KWARGS[3]), +} + + +# Adapted from https://github.com/huggingface/transformers/blob/48327c57182fdade7f7797d1eaad2d166de5c55b/src/transformers/activations.py#LL166C7-L166C22 +class ClassInstantier(OrderedDict): + def __getitem__(self, key, *args, **kwargs): + # check if any of the kwargs is inside the config class kwargs + if any([kwarg in self[key][1] for kwarg in kwargs]): + new_config_kwargs = self[key][1].copy() + new_config_kwargs.update(kwargs) + return (self[key][0], new_config_kwargs) + + return super().__getitem__(key, *args, **kwargs) + + def get_grid_parameters(self, model_list): + r""" + Returns a list of all possible combinations of the parameters in the config classes. + """ + grid_parameters = [] + for model_tuple in model_list: + model_id, lora_kwargs, prefix_tuning_kwargs, prompt_encoder_kwargs, prompt_tuning_kwargs = model_tuple + for key, value in self.items(): + if key == "lora": + # update value[1] if necessary + if lora_kwargs is not None: + value[1].update(lora_kwargs) + elif key == "prefix_tuning": + # update value[1] if necessary + if prefix_tuning_kwargs is not None: + value[1].update(prefix_tuning_kwargs) + elif key == "prompt_encoder": + # update value[1] if necessary + if prompt_encoder_kwargs is not None: + value[1].update(prompt_encoder_kwargs) + else: + # update value[1] if necessary + if prompt_tuning_kwargs is not None: + value[1].update(prompt_tuning_kwargs) + grid_parameters.append((f"test_{model_id}_{key}", model_id, value[0], value[1])) + + return grid_parameters + + +PeftTestConfigManager = ClassInstantier(CLASSES_MAPPING) diff --git a/tests/testing_utils.py b/tests/testing_utils.py new file mode 100644 index 0000000..68851ff --- /dev/null +++ b/tests/testing_utils.py @@ -0,0 +1,49 @@ +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +import torch + + +def require_torch_gpu(test_case): + """ + Decorator marking a test that requires a GPU. Will be skipped when no GPU is available. + """ + if not torch.cuda.is_available(): + return unittest.skip("test requires GPU")(test_case) + else: + return test_case + + +def require_torch_multi_gpu(test_case): + """ + Decorator marking a test that requires multiple GPUs. Will be skipped when less than 2 GPUs are available. + """ + if not torch.cuda.is_available() or torch.cuda.device_count() < 2: + return unittest.skip("test requires multiple GPUs")(test_case) + else: + return test_case + + +def require_bitsandbytes(test_case): + """ + Decorator marking a test that requires the bitsandbytes library. Will be skipped when the library is not installed. + """ + try: + import bitsandbytes # noqa: F401 + except ImportError: + return unittest.skip("test requires bitsandbytes")(test_case) + else: + return test_case