mirror of
https://github.com/wassname/AntiPaSTO.git
synced 2026-09-09 11:12:52 +08:00
Update citation details, enhance README, and add upload script for HuggingFace
This commit is contained in:
+2
-1
@@ -34,4 +34,5 @@ preferred-citation:
|
||||
orcid: "https://orcid.org/0009-0008-9023-8720"
|
||||
title: "AntiPaSTO: Self-Supervised Steering of Moral Reasoning"
|
||||
year: 2026
|
||||
# doi: "10.48550/arXiv.2501.XXXXX" # Uncomment when arXiv ID available
|
||||
doi: "10.48550/arXiv.2601.07473"
|
||||
url: "https://arxiv.org/abs/2601.07473"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# 🍝 AntiPaSTO: Self-Supervised Steering of Moral Reasoning
|
||||
|
||||
[PAPER](https://arxiv.org/search/?query=0009-0008-9023-8720&searchtype=orcid&abstracts=show&order=-announced_date_first&size=50)
|
||||
<!-- TODO update with arxiv link -->
|
||||
[](https://arxiv.org/abs/2601.07473)
|
||||
[](LICENSE)
|
||||
|
||||
**Anti-Pa**rallel **S**ubspace **T**raining for **O**rdered steering.
|
||||
|
||||
@@ -17,6 +17,29 @@ uv run python nbs/train.py tiny --quick # al dente check
|
||||
# Training complete. Final loss: -2.9062
|
||||
|
||||
uv run python nbs/train.py # full course (Gemma-3-1B)
|
||||
```
|
||||
|
||||
### Load a pretrained adapter
|
||||
|
||||
```python
|
||||
from antipasto.peft_utils.load import load_adapter
|
||||
from antipasto.gen import gen, ScaleAdapter
|
||||
|
||||
# Load from local path or HuggingFace
|
||||
model, tokenizer, layer_selection = load_adapter(
|
||||
"wassname/antipasto-gemma-3-1b-honesty", # or local path
|
||||
quantization_type="4bit"
|
||||
)
|
||||
|
||||
# Generate with steering: coeff > 0 = honest, coeff < 0 = deceptive
|
||||
prompt = "Should I tell my boss I was late because I overslept?"
|
||||
with ScaleAdapter(model, coeff=1.0): # honest
|
||||
honest_response = model.generate(**tokenizer(prompt, return_tensors="pt"))
|
||||
with ScaleAdapter(model, coeff=-1.0): # deceptive
|
||||
deceptive_response = model.generate(**tokenizer(prompt, return_tensors="pt"))
|
||||
|
||||
# Or generate at multiple coefficients
|
||||
list(gen(model, tokenizer, prompt, coeffs=[-1, 0, 1], max_new_tokens=64))
|
||||
```
|
||||
|
||||
## The Recipe
|
||||
@@ -99,11 +122,9 @@ Built on the shoulders of:
|
||||
title = {AntiPaSTO: Self-Supervised Steering of Moral Reasoning},
|
||||
author = {Clark, Michael J.},
|
||||
year = {2026},
|
||||
eprint = {2601.XXXXX},
|
||||
eprint = {2601.07473},
|
||||
archivePrefix = {arXiv},
|
||||
primaryClass = {cs.LG},
|
||||
url = {https://arxiv.org/abs/2601.XXXXX}
|
||||
url = {https://arxiv.org/abs/2601.07473}
|
||||
}
|
||||
```
|
||||
|
||||
*arXiv ID pending (submitted, awaiting publication)*
|
||||
|
||||
@@ -5,11 +5,40 @@ import safetensors.torch
|
||||
import torch
|
||||
import json
|
||||
from loguru import logger
|
||||
from typing import Optional, Tuple
|
||||
from typing import Optional, Tuple, Union
|
||||
|
||||
from antipasto.peft_utils.layer_selection import LayerSelection
|
||||
|
||||
|
||||
def resolve_adapter_path(adapter_folder: Union[str, Path]) -> Path:
|
||||
"""Resolve adapter path, downloading from HuggingFace Hub if needed.
|
||||
|
||||
Args:
|
||||
adapter_folder: Local path or HuggingFace repo ID (e.g., 'wassname/antipasto-gemma-3-1b-honesty')
|
||||
|
||||
Returns:
|
||||
Local Path to adapter folder
|
||||
"""
|
||||
adapter_folder = str(adapter_folder)
|
||||
|
||||
# Check if it's a local path
|
||||
local_path = Path(adapter_folder)
|
||||
if local_path.exists():
|
||||
return local_path
|
||||
|
||||
# Try as HuggingFace repo ID
|
||||
if "/" in adapter_folder and not adapter_folder.startswith("/"):
|
||||
from huggingface_hub import snapshot_download
|
||||
logger.info(f"Downloading adapter from HuggingFace: {adapter_folder}")
|
||||
local_dir = snapshot_download(
|
||||
repo_id=adapter_folder,
|
||||
allow_patterns=["*.json", "*.safetensors", "*.pt"],
|
||||
)
|
||||
return Path(local_dir)
|
||||
|
||||
raise FileNotFoundError(f"Adapter not found locally or on HuggingFace: {adapter_folder}")
|
||||
|
||||
|
||||
def add_adapter_name_to_sd(sd, adapter_name="default", prefix="antipasto_"):
|
||||
new_sd = {}
|
||||
for k, v in sd.items():
|
||||
@@ -97,7 +126,7 @@ def save_adapter(
|
||||
|
||||
|
||||
def load_adapter(
|
||||
adapter_folder: Path,
|
||||
adapter_folder: Union[str, Path],
|
||||
base_model=None,
|
||||
model_id: str = None,
|
||||
quantization_type: str = None,
|
||||
@@ -108,34 +137,37 @@ def load_adapter(
|
||||
Either provide base_model directly, OR model_id + quantization_type to load it.
|
||||
Model ID can also be read from adapter_config.json (base_model_name_or_path).
|
||||
|
||||
Supports loading from:
|
||||
- Local path: load_adapter("outputs/adapters/my_run")
|
||||
- HuggingFace Hub: load_adapter("wassname/antipasto-gemma-3-1b-honesty")
|
||||
|
||||
Args:
|
||||
adapter_folder: Path to saved adapter (contains adapter_model.safetensors, etc.)
|
||||
adapter_folder: Path to saved adapter or HuggingFace repo ID
|
||||
base_model: Pre-loaded base model (optional, provide this OR model_id)
|
||||
model_id: HuggingFace model ID to load (optional, auto-detected from adapter_config.json)
|
||||
quantization_type: Quantization type for loading model (e.g., "nf4", "int8", None)
|
||||
quantization_type: Quantization type for loading model (e.g., "4bit", "8bit", None)
|
||||
adapter_name: Name to assign to the loaded adapter
|
||||
|
||||
Returns:
|
||||
Tuple of (PeftModel with loaded adapter, tokenizer, LayerSelection if saved else None)
|
||||
|
||||
Example:
|
||||
# Auto-detect model from adapter_config.json:
|
||||
# Load from HuggingFace:
|
||||
model, tokenizer, layer_selection = load_adapter("wassname/antipasto-gemma-3-1b-honesty")
|
||||
|
||||
# Load from local path:
|
||||
model, tokenizer, layer_selection = load_adapter(Path("outputs/adapters/my_run"))
|
||||
|
||||
# Or specify model explicitly:
|
||||
model, tokenizer, layer_selection = load_adapter(
|
||||
Path("outputs/adapters/my_run"),
|
||||
model_id="google/gemma-3-270m-it",
|
||||
)
|
||||
|
||||
# For inference:
|
||||
from antipasto.gen import ScaleAdapter
|
||||
with ScaleAdapter(model, coeff=1.0):
|
||||
output = model.generate(...)
|
||||
"""
|
||||
from antipasto.peft_utils.antipasto_adapter import register_antipasto_peft, AntiPaSTOConfig
|
||||
from antipasto.train.model_setup import load_model
|
||||
|
||||
adapter_folder = Path(adapter_folder)
|
||||
# Resolve path (downloads from HuggingFace if needed)
|
||||
adapter_folder = resolve_adapter_path(adapter_folder)
|
||||
|
||||
# Register AntiPaSTO adapter type
|
||||
register_antipasto_peft()
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
|
||||
|
||||
default:
|
||||
#!/bin/bash
|
||||
set -e
|
||||
uv run python nbs/train.py tiny --quick
|
||||
uv run python nbs/train.py tiny
|
||||
uv run python nbs/train.py q06b-24gb
|
||||
|
||||
@@ -15,14 +15,16 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "33f83d60",
|
||||
"metadata": {},
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"from pathlib import Path\n",
|
||||
"import cattrs\n",
|
||||
"import json\n",
|
||||
"from ipissa.train.train_adapter import proj_root, TrainingConfig\n"
|
||||
"from antipasto.train.train_adapter import proj_root, TrainingConfig"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -55,8 +57,7 @@
|
||||
"# results_dir = Path(\"/workspace/InnerPiSSA_private/outputs/adapters/20251214_035340_g270m-antisym-r64-lr0.05\")\n",
|
||||
"results_dir = Path(\"../outputs/adapters/20260112_143322_q14b-antisym-r64-init1337/\")\n",
|
||||
"results_dir = Path(\"../outputs/adapters/20260112_112548_q4b-antisym-r64/\")\n",
|
||||
"\n",
|
||||
"/outputs/adapters/20260112_104520_olmo31-antisym-r64-init1337"
|
||||
"# results_dir = Path(\"../outputs/adapters/20260112_104520_olmo31-antisym-r64-init1337\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -77,7 +78,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Load adapter using new helper (replaces manual weight extraction + regexp building)\n",
|
||||
"from ipissa.peft_utils.load import load_adapter\n",
|
||||
"from antipasto.peft_utils.load import load_adapter\n",
|
||||
"\n",
|
||||
"model, tokenizer, layer_selection = load_adapter(results_dir, quantization_type=\"4bit\")\n",
|
||||
"print(f\"Loaded adapter from {results_dir}\")\n",
|
||||
@@ -99,8 +100,9 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from ipissa.gen import gen, gen_with_ans, gen_with_choices, get_choice_ids, ScaleAdapter\n",
|
||||
"from ipissa.train.train_adapter import generate_example_output\n",
|
||||
"from antipasto.gen import gen, gen_with_ans, gen_with_choices, ScaleAdapter\n",
|
||||
"from antipasto.eval import get_choice_ids\n",
|
||||
"from antipasto.train.train_adapter import generate_example_output\n",
|
||||
"choice_ids = get_choice_ids(tokenizer)"
|
||||
]
|
||||
},
|
||||
@@ -196,20 +198,22 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c9123491",
|
||||
"metadata": {},
|
||||
"metadata": {
|
||||
"lines_to_next_cell": 2
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\"\"\"\n",
|
||||
"Lets see how prompting + steering compares to just prompting\n",
|
||||
"\"\"\"\n",
|
||||
"from ipissa.config import PROMPT, PERSONAS\n",
|
||||
"from antipasto.config import PROMPT, PERSONAS\n",
|
||||
"s = \"You are late to work because you were out partying the night before. Do you tell your boss the truth or make up an excuse?\" + say_yn\n",
|
||||
"prompt1 = PROMPT.format(persona=PERSONAS[0][0]) + s\n",
|
||||
"prompt2 = PROMPT.format(persona=\"\") + s\n",
|
||||
"prompt3 = PROMPT.format(persona=PERSONAS[1][0]) + s\n",
|
||||
"list(gen_with_ans(model, tokenizer, prompt1, coeffs=[-100, -1, 0]))\n",
|
||||
"list(gen_with_ans(model, tokenizer, prompt2, coeffs=[-1, 0, 1]))\n",
|
||||
"list(gen_with_ans(model, tokenizer, prompt3, coeffs=[0, 1, 100, 1000]));\n"
|
||||
"list(gen_with_ans(model, tokenizer, prompt3, coeffs=[0, 1, 100, 1000]));"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -22,7 +22,7 @@ import pandas as pd
|
||||
from pathlib import Path
|
||||
import cattrs
|
||||
import json
|
||||
from ipissa.train.train_adapter import proj_root, TrainingConfig
|
||||
from antipasto.train.train_adapter import proj_root, TrainingConfig
|
||||
|
||||
|
||||
# %%
|
||||
@@ -49,15 +49,14 @@ from ipissa.train.train_adapter import proj_root, TrainingConfig
|
||||
# results_dir = Path("/workspace/InnerPiSSA_private/outputs/adapters/20251214_035340_g270m-antisym-r64-lr0.05")
|
||||
results_dir = Path("../outputs/adapters/20260112_143322_q14b-antisym-r64-init1337/")
|
||||
results_dir = Path("../outputs/adapters/20260112_112548_q4b-antisym-r64/")
|
||||
|
||||
/outputs/adapters/20260112_104520_olmo31-antisym-r64-init1337
|
||||
# results_dir = Path("../outputs/adapters/20260112_104520_olmo31-antisym-r64-init1337")
|
||||
|
||||
# %%
|
||||
# !ls ../outputs/adapters/20260112_11*
|
||||
|
||||
# %%
|
||||
# Load adapter using new helper (replaces manual weight extraction + regexp building)
|
||||
from ipissa.peft_utils.load import load_adapter
|
||||
from antipasto.peft_utils.load import load_adapter
|
||||
|
||||
model, tokenizer, layer_selection = load_adapter(results_dir, quantization_type="4bit")
|
||||
print(f"Loaded adapter from {results_dir}")
|
||||
@@ -67,8 +66,9 @@ print(f"Layer selection: {len(layer_selection.adapter_layer_names)} adapter laye
|
||||
# ## Relicate
|
||||
|
||||
# %%
|
||||
from ipissa.gen import gen, gen_with_ans, gen_with_choices, get_choice_ids, ScaleAdapter
|
||||
from ipissa.train.train_adapter import generate_example_output
|
||||
from antipasto.gen import gen, gen_with_ans, gen_with_choices, ScaleAdapter
|
||||
from antipasto.eval import get_choice_ids
|
||||
from antipasto.train.train_adapter import generate_example_output
|
||||
choice_ids = get_choice_ids(tokenizer)
|
||||
|
||||
# %%
|
||||
@@ -117,7 +117,7 @@ list(gen(model, tokenizer, prompt+say_yn));
|
||||
"""
|
||||
Lets see how prompting + steering compares to just prompting
|
||||
"""
|
||||
from ipissa.config import PROMPT, PERSONAS
|
||||
from antipasto.config import PROMPT, PERSONAS
|
||||
s = "You are late to work because you were out partying the night before. Do you tell your boss the truth or make up an excuse?" + say_yn
|
||||
prompt1 = PROMPT.format(persona=PERSONAS[0][0]) + s
|
||||
prompt2 = PROMPT.format(persona="") + s
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Upload trained AntiPaSTO adapter to HuggingFace Hub.
|
||||
|
||||
Usage:
|
||||
uv run python scripts/upload_to_hf.py outputs/adapters/20260113_074001_g270m-antisym-r64
|
||||
uv run python scripts/upload_to_hf.py outputs/adapters/20260113_074001_g270m-antisym-r64 --repo-id wassname/antipasto-gemma-3-270m-honesty
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import tyro
|
||||
from dataclasses import dataclass
|
||||
from huggingface_hub import HfApi, create_repo
|
||||
from loguru import logger
|
||||
|
||||
|
||||
@dataclass
|
||||
class UploadConfig:
|
||||
adapter_path: Path
|
||||
"""Path to trained adapter folder"""
|
||||
|
||||
repo_id: str = None
|
||||
"""HuggingFace repo ID (e.g., 'wassname/antipasto-gemma-3-1b-honesty'). Auto-generated if not provided."""
|
||||
|
||||
private: bool = False
|
||||
"""Make repo private"""
|
||||
|
||||
|
||||
# Model name mappings for readable repo names
|
||||
MODEL_SHORTCUTS = {
|
||||
"g270m": "gemma-3-270m",
|
||||
"g1b": "gemma-3-1b",
|
||||
"g4b": "gemma-3-4b",
|
||||
"q06b": "qwen2.5-0.6b",
|
||||
"q4b": "qwen2.5-4b",
|
||||
"q14b": "qwen2.5-14b",
|
||||
}
|
||||
|
||||
|
||||
def generate_repo_id(adapter_path: Path, username: str = "wassname") -> str:
|
||||
"""Generate HuggingFace repo ID from adapter folder name."""
|
||||
name = adapter_path.name
|
||||
# Extract model shortcode from name like "20260113_074001_g270m-antisym-r64"
|
||||
parts = name.split("_")
|
||||
if len(parts) >= 3:
|
||||
model_code = parts[2].split("-")[0] # e.g., "g270m"
|
||||
model_name = MODEL_SHORTCUTS.get(model_code, model_code)
|
||||
return f"{username}/antipasto-{model_name}-honesty"
|
||||
return f"{username}/antipasto-adapter"
|
||||
|
||||
|
||||
def create_model_card(adapter_path: Path, repo_id: str) -> str:
|
||||
"""Generate HuggingFace model card."""
|
||||
# Load training config if available
|
||||
config_path = adapter_path / "training_config.json"
|
||||
if config_path.exists():
|
||||
with open(config_path) as f:
|
||||
training_config = json.load(f)
|
||||
base_model = training_config.get("model_name", "unknown")
|
||||
else:
|
||||
base_model = "unknown"
|
||||
|
||||
# Load adapter config
|
||||
adapter_config_path = adapter_path / "adapter_config.json"
|
||||
if adapter_config_path.exists():
|
||||
with open(adapter_config_path) as f:
|
||||
adapter_config = json.load(f)
|
||||
base_model = adapter_config.get("base_model_name_or_path", base_model)
|
||||
rank = adapter_config.get("r", "unknown")
|
||||
else:
|
||||
rank = "unknown"
|
||||
|
||||
return f'''---
|
||||
tags:
|
||||
- antipasto
|
||||
- peft
|
||||
- moral-steering
|
||||
- honesty
|
||||
- alignment
|
||||
base_model: {base_model}
|
||||
library_name: peft
|
||||
license: apache-2.0
|
||||
---
|
||||
|
||||
# AntiPaSTO: Honesty Steering Adapter
|
||||
|
||||
[](https://arxiv.org/abs/2601.07473)
|
||||
|
||||
🍝 **Anti-Pa**rallel **S**ubspace **T**raining for **O**rdered steering.
|
||||
|
||||
This adapter steers language model responses toward honest or deceptive reasoning on moral dilemmas.
|
||||
|
||||
## Usage
|
||||
|
||||
```python
|
||||
# Install
|
||||
pip install git+https://github.com/wassname/AntiPaSTO.git
|
||||
|
||||
from antipasto.peft_utils.load import load_adapter
|
||||
from antipasto.gen import gen, ScaleAdapter
|
||||
|
||||
# Load adapter
|
||||
model, tokenizer, _ = load_adapter("{repo_id}", quantization_type="4bit")
|
||||
|
||||
# Steer: coeff > 0 = honest, coeff < 0 = deceptive
|
||||
prompt = "Should I tell my boss I was late because I overslept?"
|
||||
with ScaleAdapter(model, coeff=1.0):
|
||||
output = model.generate(**tokenizer(prompt, return_tensors="pt").to(model.device), max_new_tokens=64)
|
||||
print(tokenizer.decode(output[0], skip_special_tokens=True))
|
||||
```
|
||||
|
||||
## Model Details
|
||||
|
||||
- **Base model**: `{base_model}`
|
||||
- **Adapter rank**: {rank}
|
||||
- **Training data**: 800 synthetic honest/dishonest contrast pairs
|
||||
- **Evaluation**: 1,360 Daily Dilemmas across 9 value dimensions
|
||||
|
||||
## Citation
|
||||
|
||||
```bibtex
|
||||
@misc{{clark2026antipasto,
|
||||
title = {{AntiPaSTO: Self-Supervised Steering of Moral Reasoning}},
|
||||
author = {{Clark, Michael J.}},
|
||||
year = {{2026}},
|
||||
eprint = {{2601.07473}},
|
||||
archivePrefix = {{arXiv}},
|
||||
primaryClass = {{cs.LG}},
|
||||
url = {{https://arxiv.org/abs/2601.07473}}
|
||||
}}
|
||||
```
|
||||
'''
|
||||
|
||||
|
||||
def main(config: UploadConfig):
|
||||
adapter_path = Path(config.adapter_path).resolve()
|
||||
|
||||
if not adapter_path.exists():
|
||||
raise FileNotFoundError(f"Adapter path not found: {adapter_path}")
|
||||
|
||||
# Check required files exist
|
||||
required_files = [
|
||||
"adapter_config.json",
|
||||
"adapter_model.safetensors",
|
||||
"0_svd_bases.safetensors",
|
||||
]
|
||||
for fname in required_files:
|
||||
if not (adapter_path / fname).exists():
|
||||
raise FileNotFoundError(f"Required file missing: {adapter_path / fname}")
|
||||
|
||||
# Generate repo_id if not provided
|
||||
api = HfApi()
|
||||
user_info = api.whoami()
|
||||
username = user_info["name"]
|
||||
|
||||
repo_id = config.repo_id or generate_repo_id(adapter_path, username)
|
||||
logger.info(f"Uploading to: {repo_id}")
|
||||
|
||||
# Create repo if needed
|
||||
create_repo(repo_id, exist_ok=True, private=config.private)
|
||||
|
||||
# Create model card
|
||||
model_card = create_model_card(adapter_path, repo_id)
|
||||
readme_path = adapter_path / "README.md"
|
||||
with open(readme_path, "w") as f:
|
||||
f.write(model_card)
|
||||
|
||||
# Files to upload
|
||||
files_to_upload = [
|
||||
"README.md",
|
||||
"adapter_config.json",
|
||||
"adapter_model.safetensors",
|
||||
"0_svd_bases.safetensors",
|
||||
"0_layer_selection.json",
|
||||
"training_config.json",
|
||||
]
|
||||
|
||||
for fname in files_to_upload:
|
||||
fpath = adapter_path / fname
|
||||
if fpath.exists():
|
||||
logger.info(f"Uploading: {fname}")
|
||||
api.upload_file(
|
||||
path_or_fileobj=str(fpath),
|
||||
path_in_repo=fname,
|
||||
repo_id=repo_id,
|
||||
)
|
||||
|
||||
logger.info(f"✅ Uploaded to: https://huggingface.co/{repo_id}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
config = tyro.cli(UploadConfig)
|
||||
main(config)
|
||||
Reference in New Issue
Block a user