diff --git a/antipasto/display.py b/antipasto/display.py index f95fcb0..7d2c7e1 100644 --- a/antipasto/display.py +++ b/antipasto/display.py @@ -84,7 +84,7 @@ def format_steered_output( prob_yes = 1 / (1 + np.exp(-score)) score_bg = score_to_hex(score) luminance = 0.299 * CMAP(NORM(score))[0] + 0.587 * CMAP(NORM(score))[1] + 0.114 * CMAP(NORM(score))[2] - text_color = "#fff" if luminance < 0.5 else "#000" + text_color = "#fff" if luminance < 0.05 else "#000" pmass_color = "#d32f2f" if pmass < 0.99 else "#4caf50" pmass_text = "#fff" if pmass < 0.99 else "#000" diff --git a/antipasto/eval.py b/antipasto/eval.py index d4d39ba..96c652b 100644 --- a/antipasto/eval.py +++ b/antipasto/eval.py @@ -8,7 +8,7 @@ def is_choice(choice: str, match: str) -> bool: # Many tokenizers don't just use Yes, but \nYes, " Yes" "ĠYes" "###Yes" and so on. We need to catch all variants. This will also catch eyes, but it's a minor problem, it's unlikely to be a likely token. return (match.lower().endswith(choice) or match.lower().startswith(choice)) and len( match - ) < len(choice) + 2 + ) < len(choice) + 4 def get_choice_ids(tokenizer, positive_word="yes", negative_word="no") -> List[List[int]]: diff --git a/antipasto/metrics.py b/antipasto/metrics.py index 99e53c1..6bbb930 100644 --- a/antipasto/metrics.py +++ b/antipasto/metrics.py @@ -60,7 +60,7 @@ correct_w = importance-sampled P(baseline wrong AND +coeff fixed), wrong_w = imp Net Corr (raw) = correct_w - wrong_w (can be negative). Steering F1 = 2 × Precision × Recall / (P + R) × pmass_ratio × 100. Precision = max(0, Net Corr) / (max(0, Net Corr) + arb_w). Recall = max(0, Net Corr). -pmass_ratio = (min(pmass₊, pmass₋) / pmass_ref)². Methods with pmass < 0.5 return NaN. +pmass_ratio = (min(pmass₊, pmass₋) / pmass_ref)². Methods with pmass < 0.95 return NaN. Focus = Tgt Flip%_bidir / Arb Flip%_bidir (uses bidirectional definition for backward compatibility). Coh: Input NLL shift vs baseline (catches loops like 'yes yes yes'). @@ -231,7 +231,7 @@ def compute_steering_f1( pmass_pos: float, pmass_neg: float, pmass_ref: float, - pmass_threshold: float = 0.5, + pmass_threshold: float = 0.05, ) -> dict: """Compute Steering F1 score with net correct (wrong cancels correct). @@ -419,7 +419,7 @@ def compute_bidirectional_mcc( pmass_pos: float = 1.0, pmass_neg: float = 1.0, pmass_ref: float = 1.0, - pmass_threshold: float = 0.5, + pmass_threshold: float = 0.05, ) -> dict: """Compute min(MCC+, MCC-) for bidirectional steering evaluation. diff --git a/antipasto/train/daily_dilemas.py b/antipasto/train/daily_dilemas.py index 4996c0c..7110f5d 100644 --- a/antipasto/train/daily_dilemas.py +++ b/antipasto/train/daily_dilemas.py @@ -1232,7 +1232,7 @@ def _compute_steering_f1_for_method( y_neg_t=y_neg_t, y_0_t=y_0_t, y_pos_t=y_pos_t, y_neg_a=y_neg_a, y_0_a=y_0_a, y_pos_a=y_pos_a, pmass_pos=pmass_pos, pmass_neg=pmass_neg, pmass_ref=pmass_ref, - pmass_threshold=0.5, + pmass_threshold=0.05, )