wassname/lora-lite
1
0
Fork 0
mirror of https://github.com/wassname/lora-lite.git synced 2026-08-11 11:21:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(hra): add Householder Reflection Adaptation, hook-only/bnb-friendly + Qwen proof

Browse Source
This commit is contained in:
wassname
2026-04-26 17:58:56 +08:00
parent 43e620176c
commit 0d929f93b3
8 changed files with 72 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/qwen_train_probe.py
+6 -1
View File
@@ -53,6 +53,11 @@ def perturb_first_adapter(model: torch.nn.Module) -> None:
with torch.no_grad():
p.add_(0.25)
return
for name, p in model.named_parameters():
if "lora_gate" in name:
with torch.no_grad():
p.add_(0.25)
return
for name, p in model.named_parameters():
if "lora_B" in name:
with torch.no_grad():
@@ -173,7 +178,7 @@ def run_variant(args, variant: str, input_ids: torch.Tensor, labels: torch.Tenso
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--model", default="Qwen/Qwen3-0.6B")
parser.add_argument("--variants", nargs="+", default=["lora", "pissa", "delora", "ia3", "dora"])
parser.add_argument("--variants", nargs="+", default=["lora", "pissa", "delora", "ia3", "dora", "hra"])
parser.add_argument("--device", default="cuda")
parser.add_argument("--torch-dtype", default="bfloat16")
parser.add_argument("--steps", type=int, default=8)