# coding=utf-8 # Copyright 2023-present the HuggingFace Inc. team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from collections import OrderedDict from peft import ( LoraConfig, PrefixTuningConfig, PromptEncoderConfig, PromptTuningConfig, ) CONFIG_CLASSES = ( LoraConfig, PrefixTuningConfig, PromptEncoderConfig, PromptTuningConfig, ) CONFIG_TESTING_KWARGS = ( { "r": 8, "lora_alpha": 32, "target_modules": None, "lora_dropout": 0.05, "bias": "none", "task_type": "CAUSAL_LM", }, { "num_virtual_tokens": 10, "task_type": "CAUSAL_LM", }, { "num_virtual_tokens": 10, "encoder_hidden_size": 32, "task_type": "CAUSAL_LM", }, { "num_virtual_tokens": 10, "task_type": "CAUSAL_LM", }, ) CLASSES_MAPPING = { "lora": (LoraConfig, CONFIG_TESTING_KWARGS[0]), "prefix_tuning": (PrefixTuningConfig, CONFIG_TESTING_KWARGS[1]), "prompt_encoder": (PromptEncoderConfig, CONFIG_TESTING_KWARGS[2]), "prompt_tuning": (PromptTuningConfig, CONFIG_TESTING_KWARGS[3]), } # Adapted from https://github.com/huggingface/transformers/blob/48327c57182fdade7f7797d1eaad2d166de5c55b/src/transformers/activations.py#LL166C7-L166C22 class ClassInstantier(OrderedDict): def __getitem__(self, key, *args, **kwargs): # check if any of the kwargs is inside the config class kwargs if any([kwarg in self[key][1] for kwarg in kwargs]): new_config_kwargs = self[key][1].copy() new_config_kwargs.update(kwargs) return (self[key][0], new_config_kwargs) return super().__getitem__(key, *args, **kwargs) def get_grid_parameters(self, grid_parameters, filter_params_func=None): r""" Returns a list of all possible combinations of the parameters in the config classes. Args: grid_parameters (`dict`): A dictionary containing the parameters to be tested. There should be at least the key "model_ids" which contains a list of model ids to be tested. The other keys should be the name of the config class post-fixed with "_kwargs" and the value should be a dictionary containing the parameters to be tested for that config class. filter_params_func (`callable`, `optional`): A function that takes a list of tuples and returns a list of tuples. This function is used to filter out the tests that needs for example to be skipped. Returns: generated_tests (`list`): A list of tuples containing the name of the test, the model id, the config class and the config class kwargs. """ generated_tests = [] model_list = grid_parameters["model_ids"] for model_id in model_list: for key, value in self.items(): if "{}_kwargs".format(key) in grid_parameters: peft_configs = [] current_peft_config = value[1].copy() for current_key, current_value in grid_parameters[f"{key}_kwargs"].items(): for kwarg in current_value: current_peft_config.update({current_key: kwarg}) peft_configs.append(current_peft_config) else: peft_configs = [value[1].copy()] for peft_config in peft_configs: generated_tests.append((f"test_{model_id}_{key}", model_id, value[0], peft_config)) if filter_params_func is not None: generated_tests = filter_params_func(generated_tests) return generated_tests PeftTestConfigManager = ClassInstantier(CLASSES_MAPPING)