From 2aff548a7ac7b12d66cf4bd1720b5e3de41bde93 Mon Sep 17 00:00:00 2001
From: wassname <1103714+wassname@users.noreply.github.com>
Date: Sun, 8 Feb 2026 16:35:23 +0800
Subject: [PATCH] add tests

---
 README.md           | 10 ++++++----
 justfile            |  6 ++++--
 pyproject.toml      |  3 ++-
 tests/test_train.py | 40 ++++++++++++++++++++++++++++++++++++++++
 uv.lock             |  4 +++-
 5 files changed, 55 insertions(+), 8 deletions(-)
 create mode 100644 tests/test_train.py

diff --git a/README.md b/README.md
index 796c9e1..daf2450 100644
--- a/README.md
+++ b/README.md
@@ -11,9 +11,9 @@
 
 > Gradient-based honesty steering trained as an adapter on the model's own representations, not outputs. Human input: two contrasting words, no preference labels.
 
-**What does it do?** Train a single adapter (~1 hour on Gemma-3-1B) to steer honesty using just two contrasting words. At inference, dial the steering coefficient: +1 for more honest, -1 for less, 0 for baseline. One adapter, bidirectional control.
+**How it works:** Train a single adapter (~1 hour on Gemma-3-1B). At inference, dial the steering coefficient: +1 for more honest, -1 for less, 0 for baseline. One adapter, bidirectional control.
 
-**Why use it?** You want your LLM to take evals at face value and act honestly (and meta-honestly). Prompting is fragile: system prompts get ignored, jailbreaks work, and safety-trained models refuse to simulate dishonesty even when you need that for red-teaming. AntiPaSTO trains on the model's internal representations, steering what the model actually computes rather than what it says. On DailyDilemmas, it outperforms prompting by 6.9x on small models and bypasses refusal where prompting fails.
+**Why use it?** As models get more capable, eval awareness rises: models detect when they're being tested and adjust their behavior. You can't trust their outputs, their chain-of-thought, or their stated values at face value. You need a method that operates on internal representations rather than outputs, so it works even when the model is gaming the eval. AntiPaSTO steers what the model actually computes. On DailyDilemmas, it outperforms prompting by 6.9x and works where prompting triggers refusal.
 
 Applications:
 - *Combat eval awareness*: steer toward credulity and honesty so the model takes the eval at face value and gives honest answers.
@@ -29,10 +29,12 @@ Applications:
 
 ```sh
 uv sync --all-groups
-uv run python nbs/train.py tiny --quick 2>&1 | tail -300 # al dente check
-# Training complete. Final loss: -6.1250
+uv run pytest tests/test_train.py::test_train_rnd -v  # smoke test (~3min)
+uv run python nbs/train.py tiny --quick              # al dente check
 
 uv run python nbs/train.py               # full course (Gemma-3-1B)
+
+uv run python -m pytest # integration tests
 ```
 
 ### One we prepared earlier
diff --git a/justfile b/justfile
index 84a623a..485d5e9 100644
--- a/justfile
+++ b/justfile
@@ -3,9 +3,11 @@
 default:
     #!/bin/bash
     set -e
-    uv run python nbs/train.py tiny --quick
+    uv run pytest tests/test_train.py::test_train_rnd -v
+    uv run pytest tests/test_train.py::test_train_tiny -v
+
     uv run python nbs/train.py tiny
     uv run python nbs/train.py q06b-24gb
-    # uv run nbs/test_reload.py
+    
     uv run python nbs/train.py gemma1b-24gb
     uv run python nbs/train.py q4b-24gb
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index f6d6d37..ba5065d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "antipasto"
 version = "0.5.0"
-description = "Self-supervised steering of moral reasoning via antiparallel subspace training"
+description = "Self-supervised honesty steering via anti-parallel representations"
 authors = [{ name = "Michael J Clark" }]
 repository = "https://github.com/wassname/AntiPaSTO"
 license = { file = "LICENSE" }
@@ -30,6 +30,7 @@ dependencies = [
     "safetensors>=0.4.0",
     "cattrs>=25.3.0",
     "tyro>=1.0.5",
+    "tabulate>=0.9.0",
 ]
 
 [dependency-groups]
diff --git a/tests/test_train.py b/tests/test_train.py
new file mode 100644
index 0000000..2e4c95a
--- /dev/null
+++ b/tests/test_train.py
@@ -0,0 +1,40 @@
+"""Integration tests: train pipeline end-to-end."""
+import re
+import subprocess
+import sys
+
+import pytest
+
+
+def _run_train(config: str, *extra_args: str, timeout: int = 300):
+    """Run train.py with given config and assert exit code 0."""
+    cmd = [sys.executable, "nbs/train.py", config, *extra_args]
+    result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
+    assert result.returncode == 0, f"stdout:\n{result.stdout[-2000:]}\nstderr:\n{result.stderr[-2000:]}"
+    return result
+
+
+def _parse_val_loss(stdout: str) -> float:
+    """Extract validation total loss from the final losses table."""
+    # Matches the val row: "val    -9.2    +0.043    +0    -9.1"
+    match = re.search(r"^val\s+[\d.e+-]+\s+[\d.e+-]+\s+[\d.e+-]+\s+([\d.e+-]+)", stdout, re.MULTILINE)
+    assert match, f"Could not find val loss in output"
+    return float(match.group(1))
+
+
+def test_train_rnd():
+    """Smoke test: 5-layer random model, ~3min."""
+    result = _run_train("rnd")
+    assert "Saved adapter" in result.stdout
+    val_loss = _parse_val_loss(result.stdout)
+    print(f"rnd val loss: {val_loss}")
+
+
+@pytest.mark.slow
+def test_train_tiny():
+    """Larger test: gemma-3-270m-it with --quick, ~5min."""
+    result = _run_train("tiny", "--quick", timeout=600)
+    assert "Saved adapter" in result.stdout
+    val_loss = _parse_val_loss(result.stdout)
+    print(f"tiny val loss: {val_loss}")
+    assert val_loss < 0, f"Expected negative projection loss, got {val_loss}"
diff --git a/uv.lock b/uv.lock
index 4946d5a..1628d96 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1,5 +1,5 @@
 version = 1
-revision = 3
+revision = 2
 requires-python = ">=3.10"
 resolution-markers = [
     "python_full_version >= '3.12' and sys_platform == 'linux'",
@@ -217,6 +217,7 @@ dependencies = [
     { name = "scikit-learn" },
     { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
     { name = "scipy", version = "1.16.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+    { name = "tabulate" },
     { name = "torch" },
     { name = "tqdm" },
     { name = "transformers", extra = ["torch"] },
@@ -262,6 +263,7 @@ requires-dist = [
     { name = "safetensors", specifier = ">=0.4.0" },
     { name = "scikit-learn", specifier = ">=1.4.0" },
     { name = "scipy", specifier = ">=1.11.0" },
+    { name = "tabulate", specifier = ">=0.9.0" },
     { name = "torch", specifier = ">=2.1.2" },
     { name = "tqdm", specifier = ">=4.66.1" },
     { name = "transformers", extras = ["torch"], specifier = ">4.51.0" },