diff --git a/docs/RESEARCH_JOURNAL.md b/docs/RESEARCH_JOURNAL.md index 84c26d9..34391ec 100644 --- a/docs/RESEARCH_JOURNAL.md +++ b/docs/RESEARCH_JOURNAL.md @@ -11,3 +11,15 @@ Vendored reference repos into `docs/vendor` (gitignored, `just vendor` to reclon Base model for real runs: `google/gemma-3-1b-it` (gemma has more personality to steer; the alternative was a smarter-but-flatter Qwen). RTX 3090, 24 GB. **Next:** port `teacher_vec` (steering-lite + iso-KL), then the U1 filter gate. Pipeline stages currently fail fast with `NotImplementedError` pointing at the vendor module to port from. + +## Validation run on gemma-3-1b-it (3 rounds, kl_rev) — calibration too weak + +First real-model run completed end to end (`out/20260604T101347_gemma-3-1b-it_kl_rev_s42/`, log `/tmp/claude-1000/steer_heal_gemma_val.log`). Pipeline, folding, and tinymfv eval all work on a real model. + +Bug found: iso-KL calibration could not reach `target_kl=1.0`. c_star pinned at the doubling top (~25.6) with p95 KL only ~0.1 nats. The steering vector is L2-normalised, so KL ~ c^2 and ~1 nat needs c ~ O(100); steering-lite's default bracket hi (~16) is too low. + +**Interpretation:** steering was under-powered, so little trait was injected and little to heal. Symptoms: auth stuck at 0.000, care barely moved (0.307 -> 0.315 over 3 rounds), kl_rev barrier mostly below tau=0.5 (div 0.17-0.51). coherence healthy and flat (0.65-0.68); cos(v_r,v_0)=0.99/0.98 (direction stable, but a weak test under weak steering). + +**Fix:** pass `bracket=(0.1, 1024.0)` to `v.calibrate`. Re-running to confirm an interior c_star with p95 KL ~ 1.0. + +**Also to investigate:** auth=0.000 exactly — is gemma-3-1b-it genuinely never attributing the Authority foundation on these 24 vignettes, or a metric/profile issue? Check once steering is strong enough to move things. diff --git a/src/steer_heal/heal.py b/src/steer_heal/heal.py index 5abdaa6..afaefdd 100644 --- a/src/steer_heal/heal.py +++ b/src/steer_heal/heal.py @@ -63,7 +63,7 @@ def heal_round(model, tok, kept: list[dict], hist_specs: list[AdapterSpec], cfg: loss.backward() opt.step() opt.zero_grad() - logger.info(f"heal[{cfg.reg}] epoch {ep}: sft={sft.item():.3f} div={float(div):.3f}") + logger.info(f"heal[{cfg.reg}] epoch {ep}: sft={sft.item():.3f} div={div.detach().item():.3f}") spec = AdapterSpec.from_lora(lora, default_c=1.0) # CPU-resident, for the next round's history return lora, spec diff --git a/src/steer_heal/steering.py b/src/steer_heal/steering.py index 5dcd708..99ecae1 100644 --- a/src/steer_heal/steering.py +++ b/src/steer_heal/steering.py @@ -26,7 +26,10 @@ def teacher_vec(model, tok, cfg: RunConfig): logger.debug(f"--- POS[0] (trait) ---\n{pos[0]}\n--- NEG[0] (neutral) ---\n{neg[0]}") v = sl.Vector.train(model, tok, pos, neg, cfg=sl.MeanDiffC(layers=layers, normalize=True)) - v.calibrate(model, tok, target_kl=cfg.target_kl) + # Wide bracket: the vector is unit-normalised, so reaching ~1 nat p95 KL on a + # real model needs c ~ O(100) (KL ~ c^2). steering-lite's default hi (~16) is + # too low and pins c_star at the bracket top. See RESEARCH_JOURNAL 2026-06-04. + v.calibrate(model, tok, target_kl=cfg.target_kl, bracket=(0.1, 1024.0)) logger.info(f"teacher_vec: layers={layers} c_star={v.cfg.coeff:+.4f} (target_kl={cfg.target_kl})") return v