mirror of
https://github.com/wassname/lora-lite.git
synced 2026-08-18 12:10:14 +08:00
fix: corda silently ran as plain SVD; wire calibration + persist data-driven residual
The benchmark only passed calibration_data to eva, so antipasto_corda's group_init hit `if calibration_data is None: return` and every corda run was actually plain SVD. The covariance orientation never executed -- all prior corda-vs-antipasto comparisons are void. - antipasto_corda.group_init: raise on None instead of silently degrading (orientation is the variant's whole identity; fail loud). - benchmark: feed ~256 MetaMath calibration samples (IPM, per PEFT/CorDA) to corda and to cov_orient ablate; run_id now carries an __lr tag. - adapter.save/load: a data-driven group_init rewrites the frozen base residual W_res into a form init() cannot reproduce at load (it only knows the plain top-r crop). Persist those residuals in the adapter and restore them. Fixes a reload-logits mismatch that was masked while group_init never ran. - probe check: compare every saved tensor (lora_ buffers AND base residuals) against the reloaded model state. - justfile: bench-variant gains an lr_override (the core wants a tamer lr than the gain's 5e-3). Co-Authored-By: Claudypoo <noreply@anthropic.com>
This commit is contained in:
@@ -63,6 +63,7 @@ def attach(model: nn.Module, cfg: AdapterConfig, calibration_data=None, *, _skip
|
||||
attached_targets.append((name, layer, role))
|
||||
|
||||
group_init = getattr(variant, "group_init", None)
|
||||
ran_data_init = group_init is not None and not _skip_group_init and calibration_data is not None
|
||||
if group_init is not None and not _skip_group_init:
|
||||
group_init(model, attached_targets, cfg, calibration_data)
|
||||
|
||||
@@ -72,7 +73,13 @@ def attach(model: nn.Module, cfg: AdapterConfig, calibration_data=None, *, _skip
|
||||
else:
|
||||
handles.append(layer.register_forward_hook(_hook))
|
||||
|
||||
setattr(model, _ATTACHED_ATTR, {"cfg": cfg, "targets": attached_names, "handles": handles})
|
||||
# A data-driven group_init (CorDA orient, Wanda re-select) rewrites the frozen
|
||||
# base residual W_res into a form init() cannot reproduce at load time (it only
|
||||
# knows the plain top-r crop). So those residuals are part of the saved adapter.
|
||||
base_weight_keys = [f"{n}.weight" for n in attached_names] if ran_data_init else []
|
||||
setattr(model, _ATTACHED_ATTR,
|
||||
{"cfg": cfg, "targets": attached_names, "handles": handles,
|
||||
"base_weight_keys": base_weight_keys})
|
||||
return handles
|
||||
|
||||
|
||||
@@ -102,7 +109,11 @@ def save(model: nn.Module, path: str) -> None:
|
||||
state = getattr(model, _ATTACHED_ATTR, None)
|
||||
if state is None:
|
||||
raise RuntimeError("no adapter attached; call attach() first")
|
||||
sd = {k: v.detach().cpu() for k, v in model.state_dict().items() if "lora_" in k}
|
||||
full_sd = model.state_dict()
|
||||
sd = {k: v.detach().cpu() for k, v in full_sd.items() if "lora_" in k}
|
||||
# data-driven variants also persist their rewritten base residuals (see attach()).
|
||||
for wk in state.get("base_weight_keys", []):
|
||||
sd[wk] = full_sd[wk].detach().cpu()
|
||||
metadata = {"cfg": json.dumps(state["cfg"].to_dict())}
|
||||
from safetensors.torch import save_file
|
||||
save_file(sd, path, metadata=metadata)
|
||||
|
||||
@@ -92,14 +92,18 @@ class AntiPaSTOCorDA:
|
||||
def group_init(model: nn.Module, targets, cfg, calibration_data: CalibrationData | None) -> None:
|
||||
"""Re-orient each target's SVD by its input covariance C = E[x x^T].
|
||||
|
||||
Without calibration_data the plain-SVD init from init() is kept (so this
|
||||
degrades to antipasto, rotation-free).
|
||||
Covariance orientation IS this variant's identity, so calibration_data is
|
||||
mandatory -- fail loud rather than silently degrade to plain SVD (which is
|
||||
just antipasto and was the bug that made every corda run a no-op).
|
||||
|
||||
Called by attach() BEFORE any training, so the trainable g is still at its
|
||||
zero init when the basis changes -- re-orienting zero gains is a no-op, no
|
||||
re-indexing needed. Do not call group_init after training has updated g."""
|
||||
if calibration_data is None:
|
||||
return
|
||||
raise ValueError(
|
||||
"AntiPaSTOCorDA requires calibration_data (covariance orientation is "
|
||||
"its whole point); got None. Pass attach(model, cfg, calibration_data=...)."
|
||||
)
|
||||
|
||||
layers = {name: layer for name, layer, _ in targets}
|
||||
# accumulate C = sum x x^T on CPU. Peak GPU cost would otherwise be
|
||||
|
||||
Reference in New Issue
Block a user