diff --git a/nbs/analyze_diff_v2.py b/nbs/analyze_diff_v2.py index d55e0a6..86cfe90 100644 --- a/nbs/analyze_diff_v2.py +++ b/nbs/analyze_diff_v2.py @@ -44,6 +44,8 @@ Interpretation: """ # %% +import os +import sys from pathlib import Path import polars as pl @@ -57,6 +59,13 @@ from ws.data import SYCOPHANCY_TOPICS from ws.diff import load_diff from ws.steer import weight_steer +# token-efficient logging: plain message format, tqdm-safe; verbose to file +logger.remove() +logger.add(sys.stdout, level=os.environ.get("LOG_LEVEL", "INFO"), colorize=False, format="{message}") +Path("logs").mkdir(exist_ok=True) +logger.add("logs/analyze_diff_v2.verbose.log", level="DEBUG", + format="{time} | {level} | {name}:{function}:{line} - {message}") + torch.set_grad_enabled(False) @@ -386,4 +395,55 @@ print( showindex=False, ) ) -angle_df.write_csv(OUT_DIR / "analyze_diff_v2_taskdiff_vs_lmhead_angles.csv") \ No newline at end of file +angle_df.write_csv(OUT_DIR / "analyze_diff_v2_taskdiff_vs_lmhead_angles.csv") + + +# %% [markdown] +# ## Final summary (BLUF for log readers) +# +# Last ~30 lines of stdout: cue emoji + main metric, then argv/out paths, then +# a tight TSV result table for a downstream LLM/agent to read. + +# %% +active = df.filter(pl.col("layer") >= 8) +active_summary = ( + active.group_by("subspace") + .agg( + pl.col("ratio").mean().alias("mean_ratio_active"), + pl.col("ratio").max().alias("max_ratio"), + pl.col("layer").sort_by("ratio").last().alias("peak_layer"), + ) + .sort("mean_ratio_active", descending=True) +) +td_mean = active_summary.filter(pl.col("subspace") == "taskdiff")["mean_ratio_active"][0] +lm_mean = active_summary.filter(pl.col("subspace") == "lm_head_read")["mean_ratio_active"][0] +ratio_td_lm = td_mean / lm_mean if lm_mean > 0 else float("inf") +angles_active = angle_df.filter(pl.col("layer") >= 8) +max_cos_active = angles_active["max_cos"].max() if angles_active.height else float("nan") + +cue = "🟒" if (td_mean >= 5.0 and ratio_td_lm >= 3.0) else ("🟑" if td_mean >= 2.0 else "πŸ”΄") + +print() +print(f"out: {OUT_DIR}/analyze_diff_v2_concentration_summary.csv") +print(f"argv: nbs/analyze_diff_v2.py model={MODEL_ID} w={W_PATH} pcs={PCS} min_overlap={MIN_OVERLAP}") +print( + f"main metric: {cue} taskdiff_active_mean={td_mean:.2f} | " + f"lm_head_read_active_mean={lm_mean:.2f} | " + f"taskdiff/lm_head_read={ratio_td_lm:.2f} | " + f"max_cos(TaskDiff,lm_head_read)_active={max_cos_active:.2f}" +) +print() +print( + "SHOULD: cue=🟒 means taskdiff dominates lm_head_read by >=3x AND active-mean>=5; " + "🟑 means taskdiff active-mean>=2 (weak); πŸ”΄ means signal is diffuse or rides readout. " + "max_cos<0.7 confirms TaskDiff is geometrically distinct from the unembedding readout." +) +print( + tabulate( + active_summary.to_pandas(), + headers=["subspace", "mean_ratio↑", "max_ratio", "peak_layer"], + tablefmt="tsv", + floatfmt="+.2f", + showindex=False, + ) +) \ No newline at end of file diff --git a/src/ws/_log.py b/src/ws/_log.py new file mode 100644 index 0000000..8bb4eae --- /dev/null +++ b/src/ws/_log.py @@ -0,0 +1,86 @@ +"""Token-efficient loguru setup + BLUF helper. + +Call ``setup_logging("replicate")`` once at the top of an entrypoint's main(). +Stdout sink: plain, no-color, tqdm-safe, ``{message}`` only. +File sink: ``logs/.verbose.log`` at DEBUG with timestamp/location. + +Use ``final_summary(...)`` at the very end of main() to emit the standard +last-30-lines block (out: / argv: / main metric: / cue table) that a dumb +summary LLM reads first. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from typing import Any, Sequence + +from loguru import logger +from tabulate import tabulate +from tqdm.auto import tqdm + +_CONFIGURED: set[str] = set() + + +def setup_logging(name: str, log_dir: str | Path = "logs") -> Path: + """Configure loguru once per entrypoint name. Returns the verbose log path.""" + log_path = Path(log_dir) / f"{name}.verbose.log" + if name in _CONFIGURED: + return log_path + log_path.parent.mkdir(parents=True, exist_ok=True) + + logger.remove() + level = os.environ.get("LOG_LEVEL", "INFO") + # Stdout: plain, no colors, tqdm-safe + logger.add( + lambda msg: tqdm.write(msg, end=""), + level=level, + colorize=False, + format="{message}", + ) + # File: full traces for on-demand debugging + logger.add( + str(log_path), + format="{time} | {level} | {name}:{function}:{line} - {message}", + level="DEBUG", + enqueue=False, + ) + _CONFIGURED.add(name) + logger.info(f"verbose log: {log_path}") + return log_path + + +def final_summary( + *, + out: str | Path, + argv: Sequence[str] | str, + main_metric: str, + cue: str, + table_rows: Sequence[Sequence[Any]], + headers: Sequence[str], + floatfmt: str = "+.3f", +) -> None: + """Print the last-30-lines BLUF block. + + cue: '🟒' pass / '🟑' partial / 'πŸ”΄' fail. Use exactly once per run. + """ + argv_str = argv if isinstance(argv, str) else " ".join(map(str, argv)) + print() + print(f"out: {out}") + print(f"argv: {argv_str}") + print(f"main metric: {main_metric}") + rows = [[cue, *r] for r in table_rows] + print( + tabulate( + rows, + headers=["cue", *headers], + tablefmt="tsv", + floatfmt=floatfmt, + ) + ) + + +def get_argv() -> str: + """Best-effort argv reconstruction for the BLUF block.""" + return " ".join(sys.argv) diff --git a/src/ws/replicate.py b/src/ws/replicate.py index 9f38141..57071a2 100644 --- a/src/ws/replicate.py +++ b/src/ws/replicate.py @@ -17,6 +17,7 @@ from tabulate import tabulate from transformers import AutoTokenizer +from ws._log import final_summary, get_argv, setup_logging from ws.data import DataCfg, generate_pairs, load_pairs from ws.diff import compute_diff, load_base_state, load_delta, save_diff from ws.eval.sycophancy import EvalCfg, evaluate, summarize @@ -65,6 +66,7 @@ def _maybe_data(cfg: Cfg) -> Dataset: def main(cfg: Cfg) -> None: + setup_logging("replicate") ds = _maybe_data(cfg) # Train pos and neg. @@ -129,6 +131,31 @@ def main(cfg: Cfg) -> None: demo_df = phase_a2(dcfg, claims, tok) demo_df.write_csv(out_dir / "demo_guided_cot.csv") + # BLUF: headline = max margin across alpha sweep on in_dist claim + sp = summary.to_pandas() + # mean_logratio at largest positive coeff + top = sp.sort_values("coeff").iloc[-1] + bot = sp.sort_values("coeff").iloc[0] + spread = float(top["mean_logratio"]) - float(bot["mean_logratio"]) + pmin = float(sp["mean_pmass"].min()) if "mean_pmass" in sp.columns else float("nan") + cue = "🟒" if (spread > 1.0 and pmin > 0.95) else ("🟑" if spread > 0.3 else "πŸ”΄") + final_summary( + out=out_dir / "eval_summary.csv", + argv=get_argv(), + main_metric=f"logratio_spread={spread:+.3f} pmass_min={pmin:.3f}", + cue=cue, + table_rows=[[ + f"{spread:+.3f}", f"{pmin:.3f}", + f"{float(top['coeff']):+.1f}", f"{float(top['mean_logratio']):+.3f}", + cfg.behavior, cfg.adapter, cfg.model, + f"r{cfg.rank},lr{cfg.lr},ep{cfg.epochs}", + str(out_dir / "eval_summary.csv"), + ]], + headers=["logratio_spread", "pmass_min", "coeff_top", "logratio_top", + "behavior", "adapter", "model", "flags", "out"], + floatfmt="", + ) + if __name__ == "__main__": main(tyro.cli(Cfg)) diff --git a/src/ws/run_demo.py b/src/ws/run_demo.py index 0ca6384..1cabc2b 100644 --- a/src/ws/run_demo.py +++ b/src/ws/run_demo.py @@ -25,6 +25,7 @@ from peft import PeftModel from tabulate import tabulate from transformers import AutoModelForCausalLM, AutoTokenizer +from ws._log import final_summary, get_argv, setup_logging from ws.data import train_topics from ws.diff import load_diff from ws.eval.guided_cot import guided_cot_one @@ -145,6 +146,7 @@ def phase_a2(cfg: Cfg, claims: list[tuple[str, str]], tok) -> pl.DataFrame: def main(cfg: Cfg) -> None: + setup_logging("run_demo") tok = AutoTokenizer.from_pretrained(cfg.model) if tok.pad_token is None: tok.pad_token = tok.eos_token @@ -158,6 +160,30 @@ def main(cfg: Cfg) -> None: df.write_csv(out_dir / "demo_guided_cot.csv") logger.info(f"saved demo table to {out_dir / 'demo_guided_cot.csv'}") + # BLUF: in-dist margin spread across alpha + min pmass + pdf = df.to_pandas() + indist = pdf[pdf["kind"] == "in_dist"] + if len(indist): + spread = float(indist["margin"].max() - indist["margin"].min()) + else: + spread = float("nan") + pmin = float(pdf["pmass"].min()) + cue = "🟒" if (spread > 1.0 and pmin > 0.99) else ("🟑" if spread > 0.3 else "πŸ”΄") + final_summary( + out=out_dir / "demo_guided_cot.csv", + argv=get_argv(), + main_metric=f"margin_spread={spread:+.3f} pmass_min={pmin:.3f}", + cue=cue, + table_rows=[[ + f"{spread:+.3f}", f"{pmin:.3f}", + cfg.behavior, cfg.adapter, cfg.model, + f"n_think={cfg.n_think},coeffs={cfg.coeffs}", + str(out_dir / "demo_guided_cot.csv"), + ]], + headers=["margin_spread", "pmass_min", "behavior", "adapter", "model", "flags", "out"], + floatfmt="", + ) + if __name__ == "__main__": main(tyro.cli(Cfg)) diff --git a/src/ws/run_subspace.py b/src/ws/run_subspace.py index a8106a9..63395cf 100644 --- a/src/ws/run_subspace.py +++ b/src/ws/run_subspace.py @@ -17,6 +17,7 @@ import tyro from loguru import logger from tabulate import tabulate +from ws._log import final_summary, get_argv, setup_logging from ws.diff import load_base_state, load_diff from ws.subspace import alignment_table, summarize_by_kind @@ -32,6 +33,7 @@ class Cfg: def main(cfg: Cfg) -> None: + setup_logging("run_subspace") diff_path = cfg.out / cfg.behavior / cfg.adapter / "w.pt" if not diff_path.exists(): raise FileNotFoundError(f"no diff at {diff_path}; run replicate first") @@ -66,6 +68,26 @@ def main(cfg: Cfg) -> None: ) ) + # BLUF: pick the largest ratio_top across param-kinds as headline. + sp = summary.to_pandas() + best = sp.iloc[sp["mean_ratio_top"].abs().idxmax()] + rt, rw = float(best["mean_ratio_top"]), float(best["mean_ratio_weak"]) + cue = "🟒" if (rt > 1.2 or rw > 1.2) else ("🟑" if max(rt, rw) > 1.0 else "πŸ”΄") + final_summary( + out=out_dir / "subspace_summary.csv", + argv=get_argv(), + main_metric=f"max_ratio_top={rt:+.3f} max_ratio_weak={rw:+.3f} kind={best['kind']}", + cue=cue, + table_rows=[[ + f"{rt:+.3f}", f"{rw:+.3f}", best["kind"], + cfg.behavior, cfg.adapter, cfg.model, + f"k_frac={cfg.k_frac},weak_frac={cfg.weak_frac}", + str(out_dir / "subspace_summary.csv"), + ]], + headers=["ratio_top", "ratio_weak", "kind", "behavior", "adapter", "model", "flags", "out"], + floatfmt="", + ) + if __name__ == "__main__": main(tyro.cli(Cfg))