diff --git a/core/benchmark.py b/core/benchmark.py index d781626..2dc50a7 100644 --- a/core/benchmark.py +++ b/core/benchmark.py @@ -24,9 +24,8 @@ from core.scoring import ( from core.scoring import compute_detailed_distribution, compute_detailed_distribution # etc from core.separability import compute_separability_metrics from core.stability import run_stability_test, compute_iteration_stability, compute_randomized_iteration_rank_stability_by_item -from utils.stats import clamp +from utils.stats import normalize, modulate_x_by_y from utils.state import should_exit, executor -from utils.stats import normalize def process_sample(model_name: str, iteration_key: str, item_id: str, item_text: str, prompt_template: str, run_key: str, runs: Dict, runs_file: str, @@ -234,7 +233,12 @@ def finalize_scores_and_compute_judgemark(runs: dict, run_key: str, samples_data raw_emd_norm = normalize(raw_emd, 0, 4) raw_overlap_mag = run_data["separability_metrics"]["raw"]["ci99_overlap_magnitude_sum"] raw_overlap_mag_norm = normalize(raw_overlap_mag, 0, 26, False) + cohens_d_norm_raw = run_data["separability_metrics"]["raw"]["cohens_d_norm"] + # modulate ci99 overlap by cohens-d, because weak models have low overlap because they score everything in a tight range. + raw_overlap_mag_norm = modulate_x_by_y(raw_overlap_mag_norm, cohens_d_norm_raw) + raw_norm["ci99_overlap_magnitude_sum_norm"] = raw_overlap_mag_norm + raw_norm["ci99_overlap_magnitude_pct_norm"] = normalize(run_data["separability_metrics"]["raw"]["ci99_overlap_percentage_adjacent_avg"], 0, 1, False) # Range of raw model means raw_score_range = ( @@ -252,7 +256,7 @@ def finalize_scores_and_compute_judgemark(runs: dict, run_key: str, samples_data raw_separability = ( raw_norm["std_dev"] # std deviation *between* models (separability) + raw_norm["kw_stat"] # kruskal-wallis (separability) - + raw_norm["ci99_overlap_magnitude_sum_norm"] # confidence interval overlap between adjacently ranked models (separability) + + raw_norm["ci99_overlap_magnitude_pct_norm"] # confidence interval overlap between adjacently ranked models (separability) + raw_norm["raw_score_range_norm"] # range of assigned scores (separability) + run_data["separability_metrics"]["raw"]["modulated_ci95"] # average ci95 per model scored (score stability + separability) + raw_emd_norm # earth-movers distance (separability) @@ -269,7 +273,7 @@ def finalize_scores_and_compute_judgemark(runs: dict, run_key: str, samples_data "norm_correlation_with_lmsys_arena": raw_norm["kendall_tau"], "norm_std_dev_between_models": raw_norm["std_dev"], "norm_kruskall_wallis": raw_norm["kw_stat"], - "norm_ci99_adjacent_overlap": raw_norm["ci99_overlap_magnitude_sum_norm"], + "norm_ci99_adjacent_overlap": raw_norm["ci99_overlap_magnitude_pct_norm"], "norm_score_range": raw_norm["raw_score_range_norm"], "norm_intra_model_ci95": run_data["separability_metrics"]["raw"]["modulated_ci95"], "norm_earth_movers_distance": raw_emd_norm @@ -284,7 +288,11 @@ def finalize_scores_and_compute_judgemark(runs: dict, run_key: str, samples_data overlap_magnitude_norm = normalize( run_data["separability_metrics"]["calibrated"]["ci99_overlap_magnitude_sum"], 0, 26, False ) + cohens_d_norm_calibrated = run_data["separability_metrics"]["calibrated"]["cohens_d_norm"] + # modulate ci99 overlap by cohens-d, because weak models have low overlap because they score everything in a tight range. + overlap_magnitude_norm = modulate_x_by_y(overlap_magnitude_norm, cohens_d_norm_calibrated) norm["ci99_overlap_magnitude_sum_norm"] = overlap_magnitude_norm + norm["ci99_overlap_magnitude_pct_norm"] = normalize(run_data["separability_metrics"]["calibrated"]["ci99_overlap_percentage_adjacent_avg"], 0, 1, False) # Range of calibrated model means calibrated_score_range = ( @@ -304,7 +312,7 @@ def finalize_scores_and_compute_judgemark(runs: dict, run_key: str, samples_data calibrated_separability = ( norm["std_dev"] # std deviation *between* models (separability) + norm["kw_stat"] # kruskal-wallis (separability) - + norm["ci99_overlap_magnitude_sum_norm"] # confidence interval overlap between adjacently ranked models (separability) + + norm["ci99_overlap_magnitude_pct_norm"] # confidence interval overlap between adjacently ranked models (separability) + norm["calibrated_score_range_norm"] # range of assigned scores (separability) + run_data["separability_metrics"]["calibrated"]["modulated_ci95"] # average ci95 per model scored (score stability + separability) + emd_norm # earth-movers distance (separability) @@ -320,7 +328,7 @@ def finalize_scores_and_compute_judgemark(runs: dict, run_key: str, samples_data "norm_correlation_with_lmsys_arena": norm["kendall_tau"], "norm_std_dev_between_models": norm["std_dev"], "norm_kruskall_wallis": norm["kw_stat"], - "norm_ci99_adjacent_overlap": norm["ci99_overlap_magnitude_sum_norm"], + "norm_ci99_adjacent_overlap": norm["ci99_overlap_magnitude_pct_norm"], "norm_score_range": norm["calibrated_score_range_norm"], "norm_intra_model_ci95": run_data["separability_metrics"]["calibrated"]["modulated_ci95"], "norm_earth_movers_distance": norm diff --git a/core/separability.py b/core/separability.py index a585a3b..f421400 100644 --- a/core/separability.py +++ b/core/separability.py @@ -4,8 +4,8 @@ import math import statistics import numpy as np import scipy.stats -from typing import Dict, List -from utils.stats import normalize +from typing import Dict, List, Tuple +from utils.stats import normalize, modulate_x_by_y try: from scipy.stats import wasserstein_distance @@ -97,6 +97,84 @@ def compute_average_ci95(model_scores: Dict[str, List[float]]) -> float: half_widths.append(hw) return statistics.mean(half_widths) if half_widths else 0.0 + +def scale_interval(ci: Tuple[float, float], factor: float) -> Tuple[float, float]: + """ + Given an interval (low, high), expand it about its midpoint by 'factor'. + For example, if factor=1.5, the half-width becomes 1.5 times the original half-width. + """ + low, high = ci + mid = (low + high) / 2.0 + half_width = (high - low) / 2.0 + new_half = factor * half_width + return (mid - new_half, mid + new_half) + +def interval_overlap(ciA: Tuple[float, float], ciB: Tuple[float, float]) -> float: + """ + Return the length of the overlap between two intervals. + If there is no overlap, returns 0.0. + """ + return max(0.0, min(ciA[1], ciB[1]) - max(ciA[0], ciB[0])) + +def compute_adjacent_ci99_overlap_magnitude( + model_ci99: Dict[str, Tuple[float, float]], + models_sorted: List[str], + scale_factor: float, +) -> Tuple[Dict[str, float], float]: + """ + Compute the absolute overlap magnitude between adjacent models' CI99 intervals. + Each CI is first scaled by the given scale_factor. + + Returns: + - A dictionary mapping a pair key (e.g. "ModelA__ModelB") to the overlap length. + - The sum of all adjacent overlap magnitudes. + """ + adjacent_overlap_magnitude = {} + total_overlap_magnitude = 0.0 + for i in range(len(models_sorted) - 1): + mA, mB = models_sorted[i], models_sorted[i + 1] + # Scale each interval before computing overlap + scaledA = scale_interval(model_ci99[mA], scale_factor) + scaledB = scale_interval(model_ci99[mB], scale_factor) + overlap_mag = interval_overlap(scaledA, scaledB) + pair_key = f"{mA}__{mB}" + adjacent_overlap_magnitude[pair_key] = overlap_mag + total_overlap_magnitude += overlap_mag + return adjacent_overlap_magnitude, total_overlap_magnitude + +def compute_adjacent_ci99_overlap_percentage( + model_ci99: Dict[str, Tuple[float, float]], + models_sorted: List[str], +) -> Dict[str, float]: + """ + For each adjacent pair of models (ordered by descending mean), + compute the percentage overlap between their (original) CI99 intervals. + Since the intervals may have different lengths, for each pair we compute: + + perc_A = (overlap length) / (length of A's CI99) + perc_B = (overlap length) / (length of B's CI99) + + and then take the average of these two percentages. + + Returns: + A dictionary mapping a pair key (e.g. "ModelA__ModelB") to the average overlap fraction. + (e.g., 0.0 means no overlap and 1.0 means complete overlap) + """ + adjacent_overlap_percentage = {} + for i in range(len(models_sorted) - 1): + mA, mB = models_sorted[i], models_sorted[i + 1] + ciA = model_ci99[mA] + ciB = model_ci99[mB] + overlap = interval_overlap(ciA, ciB) + widthA = ciA[1] - ciA[0] + widthB = ciB[1] - ciB[0] + percA = overlap / widthA if widthA != 0 else 0.0 + percB = overlap / widthB if widthB != 0 else 0.0 + avg_perc = (percA + percB) / 2.0 + pair_key = f"{mA}__{mB}" + adjacent_overlap_percentage[pair_key] = avg_perc + return adjacent_overlap_percentage + def compute_separability_metrics( run_data: dict, scores_by_model: Dict[str, List[float]], @@ -122,6 +200,7 @@ def compute_separability_metrics( if "separability_metrics" not in run_data: run_data["separability_metrics"] = {} run_data["separability_metrics"][label] = {} + metrics_label = run_data["separability_metrics"][label] # ---------------------------------------------------------------- # 1) Basic stats: model means + 99% CI @@ -146,45 +225,29 @@ def compute_separability_metrics( overlap_count = 0 for i in range(len(models_sorted) - 1): mA, mB = models_sorted[i], models_sorted[i + 1] + # Note: ci_intervals_overlap is assumed to be defined elsewhere. overlap = ci_intervals_overlap(model_ci99[mA], model_ci99[mB]) adjacent_overlap[f"{mA}__{mB}"] = overlap if overlap: overlap_count += 1 - adj_frac_overlap = overlap_count / (len(models_sorted) - 1) if len(models_sorted) > 1 else 0.0 # ---------------------------------------------------------------- - # 3) “Magnitude” of 99% CI overlap between adjacent models - # with optional scaling factor + # 3) “Magnitude” of 99% CI overlap between adjacent models (with scaling) # ---------------------------------------------------------------- - def scale_interval(ci: tuple[float, float], factor: float) -> tuple[float, float]: - """ - Given an interval (low, high), expand it about its midpoint by 'factor'. - E.g. if factor=1.5, the half-width becomes 1.5 * (original half-width). - """ - low, high = ci - mid = (low + high) / 2.0 - half_width = (high - low) / 2.0 - new_half = factor * half_width - return (mid - new_half, mid + new_half) - - def interval_overlap(ciA: tuple[float, float], ciB: tuple[float, float]) -> float: - """Return the length of the overlap between two intervals.""" - return max(0.0, min(ciA[1], ciB[1]) - max(ciA[0], ciB[0])) - - adjacent_overlap_magnitude = {} - sum_overlap_magnitude = 0.0 - for i in range(len(models_sorted) - 1): - mA, mB = models_sorted[i], models_sorted[i + 1] - # Scale each interval before computing overlap - scaledA = scale_interval(model_ci99[mA], scale_factor) - scaledB = scale_interval(model_ci99[mB], scale_factor) - overlap_mag = interval_overlap(scaledA, scaledB) - adjacent_overlap_magnitude[f"{mA}__{mB}"] = overlap_mag - sum_overlap_magnitude += overlap_mag + adjacent_overlap_magnitude, sum_overlap_magnitude = compute_adjacent_ci99_overlap_magnitude( + model_ci99, models_sorted, scale_factor + ) # ---------------------------------------------------------------- - # 4) Single measure for Cohen’s d (average of absolute Cohen’s d across adjacent pairs) + # 4) New: Percentage overlap of the original CI99 ranges between adjacent models + # ---------------------------------------------------------------- + adjacent_overlap_percentage = compute_adjacent_ci99_overlap_percentage( + model_ci99, models_sorted + ) + + # ---------------------------------------------------------------- + # 5) Single measure for Cohen’s d (average of absolute Cohen’s d across adjacent pairs) # ---------------------------------------------------------------- d_vals = [] for i in range(len(models_sorted) - 1): @@ -194,41 +257,52 @@ def compute_separability_metrics( avg_cohens_d = sum(d_vals) / len(d_vals) if d_vals else 0.0 # ---------------------------------------------------------------- - # 5) Optional EMD across all pairs + # 6) Optional EMD across all pairs # ---------------------------------------------------------------- emd_data = compute_distributions_distance(scores_by_model) # ---------------------------------------------------------------- - # 6) Weighted or modulated average CI95 + # 7) Weighted or modulated average CI95 # ---------------------------------------------------------------- avg_ci95 = compute_average_ci95(scores_by_model) norm_ci95 = normalize(avg_ci95, 0.08, 0.45, False) norm_cohens_d = normalize(avg_cohens_d, 0, 0.4) - #modulated_ci95 = norm_ci95 * norm_cohens_d - modulated_ci95 = norm_ci95 # * norm_cohens_d + modulated_ci95 = modulate_x_by_y(norm_ci95, norm_cohens_d) # ---------------------------------------------------------------- # Store or log results # ---------------------------------------------------------------- - metrics_label = run_data["separability_metrics"][label] metrics_label["ci99_overlap_adjacent"] = adjacent_overlap metrics_label["adjacent_overlap_fraction"] = adj_frac_overlap - # New overlap magnitude stats (with scaling) metrics_label["ci99_overlap_magnitude_adjacent"] = adjacent_overlap_magnitude metrics_label["ci99_overlap_magnitude_sum"] = sum_overlap_magnitude metrics_label["ci99_overlap_scale_factor"] = scale_factor + metrics_label["ci99_overlap_percentage_adjacent"] = adjacent_overlap_percentage + ci99_overlap_percentage_adjacent_avg = 0 + if adjacent_overlap_percentage.items(): + pct_sum = 0 + for pair, perc in adjacent_overlap_percentage.items(): + pct_sum += perc + ci99_overlap_percentage_adjacent_avg = pct_sum / len(adjacent_overlap_percentage) + + metrics_label["ci99_overlap_percentage_adjacent_avg"] = ci99_overlap_percentage_adjacent_avg + metrics_label["average_cohens_d_adjacent"] = avg_cohens_d + metrics_label["cohens_d_norm"] = norm_cohens_d metrics_label["emd"] = emd_data metrics_label["average_ci95"] = avg_ci95 - metrics_label["modulated_ci95"] = modulated_ci95 + metrics_label["modulated_ci95"] = modulated_ci95 # Logging summary logging.info(f"\n--- {label.upper()} SEPARABILITY METRICS ---") logging.info(f"Adjacent 99% CI Overlap fraction: {adj_frac_overlap:.3f}") logging.info(f"Sum of adjacent 99% CI Overlap magnitude (scale={scale_factor}): " f"{sum_overlap_magnitude:.3f}") + logging.info(f"CI99 Overlap pct: " + f"{ci99_overlap_percentage_adjacent_avg:.3f}") + logging.info(f"Avg. |Cohen's d| for adjacent pairs: {avg_cohens_d:.3f}") logging.info(f"Average EMD across all pairs: {emd_data['average']:.3f}") - logging.info(f"Avg. CI95 half-width: {avg_ci95:.3f} (modulated: {modulated_ci95:.3f})") \ No newline at end of file + logging.info(f"Avg. CI95 half-width: {avg_ci95:.3f} (modulated: {modulated_ci95:.3f})") diff --git a/utils/stats.py b/utils/stats.py index ad83e0f..daacfc0 100644 --- a/utils/stats.py +++ b/utils/stats.py @@ -8,4 +8,32 @@ def normalize(val, min_val, max_val, bigger_is_better=True): norm = (val - min_val) / (max_val - min_val) if not bigger_is_better: norm = 1.0 - norm - return clamp(norm) \ No newline at end of file + return clamp(norm) + +def modulate_x_by_y(x, y): + """ + Modulate x so that it is sharply reduced if y is < 0.3, tapering to no effect as y approaches 1. + + Use case example: + Modulate a confidence interval based on Cohen's d value. + + Weak judges have low ci95 ranges for each model they score. + For a strong judge that's a good thing, but for the weak judge + it just means they score *everything* within a narrow band. + + Here we compensate for this by modulating the ci95 range by a + different measure of separability (cohen's d). When ci95 is + large but cohen's d is small, the modulated value is also small. + """ + + def modulation_factor(d): + if d <= 0.3: + # Steeper rise to reach ~0.95 by 0.3 + return 3.17 * d * (1 - 0.15 * d) + else: + # Smooth curve approaching 1 after 0.3 + t = (d - 0.3) / 0.7 # normalize remaining part to 0-1 + base = 3.17 * 0.3 * (1 - 0.15 * 0.3) # value at d=0.3 + return base + (1 - base) * (t * (2 - t)) + + return x * modulation_factor(y) \ No newline at end of file