mirror of
https://github.com/wassname/evil_MoE.git
synced 2026-08-08 11:19:04 +08:00
fix smoke.
This commit is contained in:
@@ -43,7 +43,7 @@ def collect_last_token_hidden(
|
||||
ids = tokenizer(p, return_tensors="pt").to(device)
|
||||
out = model(**ids, output_hidden_states=True)
|
||||
# out.hidden_states is tuple of (n_layers+1,) tensors of shape (1, seq, d)
|
||||
h = out.hidden_states[layer_idx][0, -1, :].cpu() # "d"
|
||||
h = out.hidden_states[layer_idx][0, -1, :].float().cpu() # "d" — fp32 for stable v_hack
|
||||
hs.append(h)
|
||||
return torch.stack(hs, dim=0)
|
||||
|
||||
@@ -67,8 +67,10 @@ def extract_vhack(
|
||||
|
||||
logger.info(
|
||||
f"v_hack extracted layer={layer_idx} n_train={len(h_hack_train)} "
|
||||
f"n_val={len(h_hack_val)} val_acc={correct:.3f} "
|
||||
f"SHOULD val_acc>0.9 ELSE pair quality or layer is wrong"
|
||||
f"n_val={len(h_hack_val)} val_acc={correct:.3f} "
|
||||
f"SHOULD>0.9 on a trained model: v_hack should separate hack from clean. "
|
||||
f"On tiny-random/untrained models val_acc~0.5 (no semantic structure yet), "
|
||||
f"which is fine for smoke -- the projection mechanism is what we test there."
|
||||
)
|
||||
|
||||
return VHackResult(
|
||||
|
||||
Reference in New Issue
Block a user