From ce61e2452ae623f6cf41c755b51f35f1aff30dc4 Mon Sep 17 00:00:00 2001 From: Zhang Date: Wed, 29 Mar 2023 21:03:48 -0400 Subject: [PATCH] define the resize function --- src/peft/tuners/adalora.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/peft/tuners/adalora.py b/src/peft/tuners/adalora.py index 648c05a..ae5fe20 100644 --- a/src/peft/tuners/adalora.py +++ b/src/peft/tuners/adalora.py @@ -225,6 +225,15 @@ class AdaLoraModel(LoraModel): return new_module + def resize_modules_by_rank_pattern(self, rank_pattern): + for name,rank_idx in rank_pattern.items(): + key = ".".join(name.split(".")[1:-1]) + parent, target, target_name = self._get_submodules(key) + + new_module = self._prepare_new_module(target, rank_idx) + self._replace_module(parent, target_name, new_module, target) + + def update_and_allocate(self, global_step): # Update the importance score and allocate the budget if global_step < self.peft_config.total_step - self.peft_config.tfinal: @@ -233,14 +242,17 @@ class AdaLoraModel(LoraModel): elif global_step == self.peft_config.total_step - self.peft_config.tfinal: budget, rank_pattern = self.rankallocator.update_and_allocate(self, global_step, force_mask=True) - for name,rank_idx in rank_pattern.items(): - key = ".".join(name.split(".")[1:-1]) - parent, target, target_name = self._get_submodules(key) + self.resize_modules_by_rank_pattern(rank_pattern) - new_module = self._prepare_new_module(target, rank_idx) - self._replace_module(parent, target_name, new_module, target) + # for name,rank_idx in rank_pattern.items(): + # key = ".".join(name.split(".")[1:-1]) + # parent, target, target_name = self._get_submodules(key) + + # new_module = self._prepare_new_module(target, rank_idx) + # self._replace_module(parent, target_name, new_module, target) print("Finalize the rank pattern.") self.rankallocator.reset_ipt() + self.rank_pattern = rank_pattern # Pass the function and do forward propagation else: return None