logs

nbs/analyze_diff_v2.py

@@ -44,6 +44,8 @@ Interpretation:
 """
 
 # %%
+import os
+import sys
 from pathlib import Path
 
 import polars as pl
@@ -57,6 +59,13 @@ from ws.data import SYCOPHANCY_TOPICS
 from ws.diff import load_diff
 from ws.steer import weight_steer
 
+# token-efficient logging: plain message format, tqdm-safe; verbose to file
+logger.remove()
+logger.add(sys.stdout, level=os.environ.get("LOG_LEVEL", "INFO"), colorize=False, format="{message}")
+Path("logs").mkdir(exist_ok=True)
+logger.add("logs/analyze_diff_v2.verbose.log", level="DEBUG",
+           format="{time} | {level} | {name}:{function}:{line} - {message}")
+
 torch.set_grad_enabled(False)
 
 
@@ -386,4 +395,55 @@ print(
         showindex=False,
     )
 )
-angle_df.write_csv(OUT_DIR / "analyze_diff_v2_taskdiff_vs_lmhead_angles.csv")
+angle_df.write_csv(OUT_DIR / "analyze_diff_v2_taskdiff_vs_lmhead_angles.csv")
+
+
+# %% [markdown]
+# ## Final summary (BLUF for log readers)
+#
+# Last ~30 lines of stdout: cue emoji + main metric, then argv/out paths, then
+# a tight TSV result table for a downstream LLM/agent to read.
+
+# %%
+active = df.filter(pl.col("layer") >= 8)
+active_summary = (
+    active.group_by("subspace")
+    .agg(
+        pl.col("ratio").mean().alias("mean_ratio_active"),
+        pl.col("ratio").max().alias("max_ratio"),
+        pl.col("layer").sort_by("ratio").last().alias("peak_layer"),
+    )
+    .sort("mean_ratio_active", descending=True)
+)
+td_mean = active_summary.filter(pl.col("subspace") == "taskdiff")["mean_ratio_active"][0]
+lm_mean = active_summary.filter(pl.col("subspace") == "lm_head_read")["mean_ratio_active"][0]
+ratio_td_lm = td_mean / lm_mean if lm_mean > 0 else float("inf")
+angles_active = angle_df.filter(pl.col("layer") >= 8)
+max_cos_active = angles_active["max_cos"].max() if angles_active.height else float("nan")
+
+cue = "🟢" if (td_mean >= 5.0 and ratio_td_lm >= 3.0) else ("🟡" if td_mean >= 2.0 else "🔴")
+
+print()
+print(f"out: {OUT_DIR}/analyze_diff_v2_concentration_summary.csv")
+print(f"argv: nbs/analyze_diff_v2.py model={MODEL_ID} w={W_PATH} pcs={PCS} min_overlap={MIN_OVERLAP}")
+print(
+    f"main metric: {cue} taskdiff_active_mean={td_mean:.2f} | "
+    f"lm_head_read_active_mean={lm_mean:.2f} | "
+    f"taskdiff/lm_head_read={ratio_td_lm:.2f} | "
+    f"max_cos(TaskDiff,lm_head_read)_active={max_cos_active:.2f}"
+)
+print()
+print(
+    "SHOULD: cue=🟢 means taskdiff dominates lm_head_read by >=3x AND active-mean>=5; "
+    "🟡 means taskdiff active-mean>=2 (weak); 🔴 means signal is diffuse or rides readout. "
+    "max_cos<0.7 confirms TaskDiff is geometrically distinct from the unembedding readout."
+)
+print(
+    tabulate(
+        active_summary.to_pandas(),
+        headers=["subspace", "mean_ratio↑", "max_ratio", "peak_layer"],
+        tablefmt="tsv",
+        floatfmt="+.2f",
+        showindex=False,
+    )
+)

src/ws/_log.py

@@ -0,0 +1,86 @@
+"""Token-efficient loguru setup + BLUF helper.
+
+Call ``setup_logging("replicate")`` once at the top of an entrypoint's main().
+Stdout sink: plain, no-color, tqdm-safe, ``{message}`` only.
+File sink: ``logs/<name>.verbose.log`` at DEBUG with timestamp/location.
+
+Use ``final_summary(...)`` at the very end of main() to emit the standard
+last-30-lines block (out: / argv: / main metric: / cue table) that a dumb
+summary LLM reads first.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+from typing import Any, Sequence
+
+from loguru import logger
+from tabulate import tabulate
+from tqdm.auto import tqdm
+
+_CONFIGURED: set[str] = set()
+
+
+def setup_logging(name: str, log_dir: str | Path = "logs") -> Path:
+    """Configure loguru once per entrypoint name. Returns the verbose log path."""
+    log_path = Path(log_dir) / f"{name}.verbose.log"
+    if name in _CONFIGURED:
+        return log_path
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+
+    logger.remove()
+    level = os.environ.get("LOG_LEVEL", "INFO")
+    # Stdout: plain, no colors, tqdm-safe
+    logger.add(
+        lambda msg: tqdm.write(msg, end=""),
+        level=level,
+        colorize=False,
+        format="{message}",
+    )
+    # File: full traces for on-demand debugging
+    logger.add(
+        str(log_path),
+        format="{time} | {level} | {name}:{function}:{line} - {message}",
+        level="DEBUG",
+        enqueue=False,
+    )
+    _CONFIGURED.add(name)
+    logger.info(f"verbose log: {log_path}")
+    return log_path
+
+
+def final_summary(
+    *,
+    out: str | Path,
+    argv: Sequence[str] | str,
+    main_metric: str,
+    cue: str,
+    table_rows: Sequence[Sequence[Any]],
+    headers: Sequence[str],
+    floatfmt: str = "+.3f",
+) -> None:
+    """Print the last-30-lines BLUF block.
+
+    cue: '🟢' pass / '🟡' partial / '🔴' fail. Use exactly once per run.
+    """
+    argv_str = argv if isinstance(argv, str) else " ".join(map(str, argv))
+    print()
+    print(f"out: {out}")
+    print(f"argv: {argv_str}")
+    print(f"main metric: {main_metric}")
+    rows = [[cue, *r] for r in table_rows]
+    print(
+        tabulate(
+            rows,
+            headers=["cue", *headers],
+            tablefmt="tsv",
+            floatfmt=floatfmt,
+        )
+    )
+
+
+def get_argv() -> str:
+    """Best-effort argv reconstruction for the BLUF block."""
+    return " ".join(sys.argv)

src/ws/replicate.py

@@ -17,6 +17,7 @@ from tabulate import tabulate
 
 from transformers import AutoTokenizer
 
+from ws._log import final_summary, get_argv, setup_logging
 from ws.data import DataCfg, generate_pairs, load_pairs
 from ws.diff import compute_diff, load_base_state, load_delta, save_diff
 from ws.eval.sycophancy import EvalCfg, evaluate, summarize
@@ -65,6 +66,7 @@ def _maybe_data(cfg: Cfg) -> Dataset:
 
 
 def main(cfg: Cfg) -> None:
+    setup_logging("replicate")
     ds = _maybe_data(cfg)
 
     # Train pos and neg.
@@ -129,6 +131,31 @@ def main(cfg: Cfg) -> None:
     demo_df = phase_a2(dcfg, claims, tok)
     demo_df.write_csv(out_dir / "demo_guided_cot.csv")
 
+    # BLUF: headline = max margin across alpha sweep on in_dist claim
+    sp = summary.to_pandas()
+    # mean_logratio at largest positive coeff
+    top = sp.sort_values("coeff").iloc[-1]
+    bot = sp.sort_values("coeff").iloc[0]
+    spread = float(top["mean_logratio"]) - float(bot["mean_logratio"])
+    pmin = float(sp["mean_pmass"].min()) if "mean_pmass" in sp.columns else float("nan")
+    cue = "🟢" if (spread > 1.0 and pmin > 0.95) else ("🟡" if spread > 0.3 else "🔴")
+    final_summary(
+        out=out_dir / "eval_summary.csv",
+        argv=get_argv(),
+        main_metric=f"logratio_spread={spread:+.3f} pmass_min={pmin:.3f}",
+        cue=cue,
+        table_rows=[[
+            f"{spread:+.3f}", f"{pmin:.3f}",
+            f"{float(top['coeff']):+.1f}", f"{float(top['mean_logratio']):+.3f}",
+            cfg.behavior, cfg.adapter, cfg.model,
+            f"r{cfg.rank},lr{cfg.lr},ep{cfg.epochs}",
+            str(out_dir / "eval_summary.csv"),
+        ]],
+        headers=["logratio_spread", "pmass_min", "coeff_top", "logratio_top",
+                 "behavior", "adapter", "model", "flags", "out"],
+        floatfmt="",
+    )
+
 
 if __name__ == "__main__":
     main(tyro.cli(Cfg))

src/ws/run_demo.py

@@ -25,6 +25,7 @@ from peft import PeftModel
 from tabulate import tabulate
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
+from ws._log import final_summary, get_argv, setup_logging
 from ws.data import train_topics
 from ws.diff import load_diff
 from ws.eval.guided_cot import guided_cot_one
@@ -145,6 +146,7 @@ def phase_a2(cfg: Cfg, claims: list[tuple[str, str]], tok) -> pl.DataFrame:
 
 
 def main(cfg: Cfg) -> None:
+    setup_logging("run_demo")
     tok = AutoTokenizer.from_pretrained(cfg.model)
     if tok.pad_token is None:
         tok.pad_token = tok.eos_token
@@ -158,6 +160,30 @@ def main(cfg: Cfg) -> None:
     df.write_csv(out_dir / "demo_guided_cot.csv")
     logger.info(f"saved demo table to {out_dir / 'demo_guided_cot.csv'}")
 
+    # BLUF: in-dist margin spread across alpha + min pmass
+    pdf = df.to_pandas()
+    indist = pdf[pdf["kind"] == "in_dist"]
+    if len(indist):
+        spread = float(indist["margin"].max() - indist["margin"].min())
+    else:
+        spread = float("nan")
+    pmin = float(pdf["pmass"].min())
+    cue = "🟢" if (spread > 1.0 and pmin > 0.99) else ("🟡" if spread > 0.3 else "🔴")
+    final_summary(
+        out=out_dir / "demo_guided_cot.csv",
+        argv=get_argv(),
+        main_metric=f"margin_spread={spread:+.3f} pmass_min={pmin:.3f}",
+        cue=cue,
+        table_rows=[[
+            f"{spread:+.3f}", f"{pmin:.3f}",
+            cfg.behavior, cfg.adapter, cfg.model,
+            f"n_think={cfg.n_think},coeffs={cfg.coeffs}",
+            str(out_dir / "demo_guided_cot.csv"),
+        ]],
+        headers=["margin_spread", "pmass_min", "behavior", "adapter", "model", "flags", "out"],
+        floatfmt="",
+    )
+
 
 if __name__ == "__main__":
     main(tyro.cli(Cfg))

src/ws/run_subspace.py

@@ -17,6 +17,7 @@ import tyro
 from loguru import logger
 from tabulate import tabulate
 
+from ws._log import final_summary, get_argv, setup_logging
 from ws.diff import load_base_state, load_diff
 from ws.subspace import alignment_table, summarize_by_kind
 
@@ -32,6 +33,7 @@ class Cfg:
 
 
 def main(cfg: Cfg) -> None:
+    setup_logging("run_subspace")
     diff_path = cfg.out / cfg.behavior / cfg.adapter / "w.pt"
     if not diff_path.exists():
         raise FileNotFoundError(f"no diff at {diff_path}; run replicate first")
@@ -66,6 +68,26 @@ def main(cfg: Cfg) -> None:
         )
     )
 
+    # BLUF: pick the largest ratio_top across param-kinds as headline.
+    sp = summary.to_pandas()
+    best = sp.iloc[sp["mean_ratio_top"].abs().idxmax()]
+    rt, rw = float(best["mean_ratio_top"]), float(best["mean_ratio_weak"])
+    cue = "🟢" if (rt > 1.2 or rw > 1.2) else ("🟡" if max(rt, rw) > 1.0 else "🔴")
+    final_summary(
+        out=out_dir / "subspace_summary.csv",
+        argv=get_argv(),
+        main_metric=f"max_ratio_top={rt:+.3f} max_ratio_weak={rw:+.3f} kind={best['kind']}",
+        cue=cue,
+        table_rows=[[
+            f"{rt:+.3f}", f"{rw:+.3f}", best["kind"],
+            cfg.behavior, cfg.adapter, cfg.model,
+            f"k_frac={cfg.k_frac},weak_frac={cfg.weak_frac}",
+            str(out_dir / "subspace_summary.csv"),
+        ]],
+        headers=["ratio_top", "ratio_weak", "kind", "behavior", "adapter", "model", "flags", "out"],
+        floatfmt="",
+    )
+
 
 if __name__ == "__main__":
     main(tyro.cli(Cfg))