wassname/peft
1
0
Fork 0
mirror of https://github.com/wassname/peft.git synced 2026-09-09 11:28:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixing 🐛

Browse Source
This commit is contained in:
Sourab Mangrulkar
2023-04-06 21:17:54 +05:30
parent 3258b709a3
commit d5feb8b787
1 changed files with 5 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/peft/tuners/adalora.py
+5 -11
View File
@@ -243,7 +243,7 @@ class AdaLoraModel(LoraModel):
rank = rank_idx.sum().item()
else:
raise ValueError("Unexcepted type of rank_idx")
key = ".".join(name.split(".")[0:-1])
key = ".".join(name.split(".")[0:-2])
_, target, _ = _get_submodules(self.model, key)
lora_E_weights = target.lora_E[adapter_name][rank_idx]
lora_A_weights = target.lora_A[adapter_name][rank_idx]
@@ -320,19 +320,13 @@ class AdaLoraLayer(LoraLayer):
# Actual trainable parameters
if r > 0:
# Right singular vectors
self.lora_A.update(
nn.ParameterDict({adapter_name: nn.Parameter(self.weight.new_zeros((r, self.in_features)))})
)
self.lora_A.update(nn.ParameterDict({adapter_name: nn.Parameter(torch.zeros(r, self.in_features))}))
# Singular values
self.lora_E.update(nn.ParameterDict({adapter_name: nn.Parameter(self.weight.new_zeros(r, 1))}))
self.lora_E.update(nn.ParameterDict({adapter_name: nn.Parameter(torch.zeros(r, 1))}))
# Left singular vectors
self.lora_B.update(
nn.ParameterDict({adapter_name: nn.Parameter(self.weight.new_zeros((self.out_features, r)))})
)
self.lora_B.update(nn.ParameterDict({adapter_name: nn.Parameter(torch.zeros(self.out_features, r))}))
# The current rank
self.ranknum.update(
nn.ParameterDict({adapter_name: nn.Parameter(self.weight.new_zeros(1), requires_grad=False)})
)
self.ranknum.update(nn.ParameterDict({adapter_name: nn.Parameter(torch.zeros(1), requires_grad=False)}))
self.ranknum[adapter_name].data.fill_(float(r))
self.ranknum[adapter_name].requires_grad = False
self.scaling[adapter_name] = lora_alpha if lora_alpha > 0 else float(r)