wassname/peft
1
0
Fork 0
mirror of https://github.com/wassname/peft.git synced 2026-09-09 11:28:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix autocast issue (#121)

Browse Source
This commit is contained in:
Younes Belkada
2023-02-23 09:17:40 +01:00
committed by GitHub
parent 75925b1aae
commit a3537160dc
1 changed files with 11 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/peft/tuners/lora.py
+11 -1
View File
@@ -494,8 +494,18 @@ if is_bnb_available():
def forward(self, x: torch.Tensor):
result = super().forward(x)
if self.disable_adapters:
return result
elif self.r > 0:
result += self.lora_B(self.lora_A(self.lora_dropout(x))) * self.scaling
if not torch.is_autocast_enabled():
expected_dtype = result.dtype
if x.dtype != torch.float32:
x = x.float()
output = self.lora_B(self.lora_A(self.lora_dropout(x))).to(expected_dtype) * self.scaling
result += output
else:
output = self.lora_B(self.lora_A(self.lora_dropout(x))) * self.scaling
result += output
return result