From efbbcd74d502beb711e7b222083b23023de2afdf Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Tue, 13 Jan 2026 12:07:02 +0800 Subject: [PATCH] Update citation details, enhance README, and add upload script for HuggingFace --- CITATION.cff | 3 +- README.md | 33 ++++-- antipasto/peft_utils/load.py | 56 +++++++--- justfile | 2 + nbs/talk_to_checkpoint.ipynb | 24 +++-- nbs/talk_to_checkpoint.py | 14 +-- scripts/upload_to_hf.py | 193 +++++++++++++++++++++++++++++++++++ 7 files changed, 289 insertions(+), 36 deletions(-) create mode 100644 scripts/upload_to_hf.py diff --git a/CITATION.cff b/CITATION.cff index 35070c1..2b0f4b9 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -34,4 +34,5 @@ preferred-citation: orcid: "https://orcid.org/0009-0008-9023-8720" title: "AntiPaSTO: Self-Supervised Steering of Moral Reasoning" year: 2026 - # doi: "10.48550/arXiv.2501.XXXXX" # Uncomment when arXiv ID available + doi: "10.48550/arXiv.2601.07473" + url: "https://arxiv.org/abs/2601.07473" diff --git a/README.md b/README.md index acc62f3..0c5036c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # 🍝 AntiPaSTO: Self-Supervised Steering of Moral Reasoning -[PAPER](https://arxiv.org/search/?query=0009-0008-9023-8720&searchtype=orcid&abstracts=show&order=-announced_date_first&size=50) - +[![arXiv](https://img.shields.io/badge/arXiv-2601.07473-b31b1b.svg)](https://arxiv.org/abs/2601.07473) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) **Anti-Pa**rallel **S**ubspace **T**raining for **O**rdered steering. @@ -17,6 +17,29 @@ uv run python nbs/train.py tiny --quick # al dente check # Training complete. Final loss: -2.9062 uv run python nbs/train.py # full course (Gemma-3-1B) +``` + +### Load a pretrained adapter + +```python +from antipasto.peft_utils.load import load_adapter +from antipasto.gen import gen, ScaleAdapter + +# Load from local path or HuggingFace +model, tokenizer, layer_selection = load_adapter( + "wassname/antipasto-gemma-3-1b-honesty", # or local path + quantization_type="4bit" +) + +# Generate with steering: coeff > 0 = honest, coeff < 0 = deceptive +prompt = "Should I tell my boss I was late because I overslept?" +with ScaleAdapter(model, coeff=1.0): # honest + honest_response = model.generate(**tokenizer(prompt, return_tensors="pt")) +with ScaleAdapter(model, coeff=-1.0): # deceptive + deceptive_response = model.generate(**tokenizer(prompt, return_tensors="pt")) + +# Or generate at multiple coefficients +list(gen(model, tokenizer, prompt, coeffs=[-1, 0, 1], max_new_tokens=64)) ``` ## The Recipe @@ -99,11 +122,9 @@ Built on the shoulders of: title = {AntiPaSTO: Self-Supervised Steering of Moral Reasoning}, author = {Clark, Michael J.}, year = {2026}, - eprint = {2601.XXXXX}, + eprint = {2601.07473}, archivePrefix = {arXiv}, primaryClass = {cs.LG}, - url = {https://arxiv.org/abs/2601.XXXXX} + url = {https://arxiv.org/abs/2601.07473} } ``` - -*arXiv ID pending (submitted, awaiting publication)* diff --git a/antipasto/peft_utils/load.py b/antipasto/peft_utils/load.py index 8e7a6aa..7662cb8 100644 --- a/antipasto/peft_utils/load.py +++ b/antipasto/peft_utils/load.py @@ -5,11 +5,40 @@ import safetensors.torch import torch import json from loguru import logger -from typing import Optional, Tuple +from typing import Optional, Tuple, Union from antipasto.peft_utils.layer_selection import LayerSelection +def resolve_adapter_path(adapter_folder: Union[str, Path]) -> Path: + """Resolve adapter path, downloading from HuggingFace Hub if needed. + + Args: + adapter_folder: Local path or HuggingFace repo ID (e.g., 'wassname/antipasto-gemma-3-1b-honesty') + + Returns: + Local Path to adapter folder + """ + adapter_folder = str(adapter_folder) + + # Check if it's a local path + local_path = Path(adapter_folder) + if local_path.exists(): + return local_path + + # Try as HuggingFace repo ID + if "/" in adapter_folder and not adapter_folder.startswith("/"): + from huggingface_hub import snapshot_download + logger.info(f"Downloading adapter from HuggingFace: {adapter_folder}") + local_dir = snapshot_download( + repo_id=adapter_folder, + allow_patterns=["*.json", "*.safetensors", "*.pt"], + ) + return Path(local_dir) + + raise FileNotFoundError(f"Adapter not found locally or on HuggingFace: {adapter_folder}") + + def add_adapter_name_to_sd(sd, adapter_name="default", prefix="antipasto_"): new_sd = {} for k, v in sd.items(): @@ -97,7 +126,7 @@ def save_adapter( def load_adapter( - adapter_folder: Path, + adapter_folder: Union[str, Path], base_model=None, model_id: str = None, quantization_type: str = None, @@ -108,34 +137,37 @@ def load_adapter( Either provide base_model directly, OR model_id + quantization_type to load it. Model ID can also be read from adapter_config.json (base_model_name_or_path). + Supports loading from: + - Local path: load_adapter("outputs/adapters/my_run") + - HuggingFace Hub: load_adapter("wassname/antipasto-gemma-3-1b-honesty") + Args: - adapter_folder: Path to saved adapter (contains adapter_model.safetensors, etc.) + adapter_folder: Path to saved adapter or HuggingFace repo ID base_model: Pre-loaded base model (optional, provide this OR model_id) model_id: HuggingFace model ID to load (optional, auto-detected from adapter_config.json) - quantization_type: Quantization type for loading model (e.g., "nf4", "int8", None) + quantization_type: Quantization type for loading model (e.g., "4bit", "8bit", None) adapter_name: Name to assign to the loaded adapter Returns: Tuple of (PeftModel with loaded adapter, tokenizer, LayerSelection if saved else None) Example: - # Auto-detect model from adapter_config.json: + # Load from HuggingFace: + model, tokenizer, layer_selection = load_adapter("wassname/antipasto-gemma-3-1b-honesty") + + # Load from local path: model, tokenizer, layer_selection = load_adapter(Path("outputs/adapters/my_run")) - # Or specify model explicitly: - model, tokenizer, layer_selection = load_adapter( - Path("outputs/adapters/my_run"), - model_id="google/gemma-3-270m-it", - ) - # For inference: + from antipasto.gen import ScaleAdapter with ScaleAdapter(model, coeff=1.0): output = model.generate(...) """ from antipasto.peft_utils.antipasto_adapter import register_antipasto_peft, AntiPaSTOConfig from antipasto.train.model_setup import load_model - adapter_folder = Path(adapter_folder) + # Resolve path (downloads from HuggingFace if needed) + adapter_folder = resolve_adapter_path(adapter_folder) # Register AntiPaSTO adapter type register_antipasto_peft() diff --git a/justfile b/justfile index bf22f58..84a623a 100644 --- a/justfile +++ b/justfile @@ -1,6 +1,8 @@ default: + #!/bin/bash + set -e uv run python nbs/train.py tiny --quick uv run python nbs/train.py tiny uv run python nbs/train.py q06b-24gb diff --git a/nbs/talk_to_checkpoint.ipynb b/nbs/talk_to_checkpoint.ipynb index 3e6da3f..29e701b 100644 --- a/nbs/talk_to_checkpoint.ipynb +++ b/nbs/talk_to_checkpoint.ipynb @@ -15,14 +15,16 @@ "cell_type": "code", "execution_count": null, "id": "33f83d60", - "metadata": {}, + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [], "source": [ "import pandas as pd\n", "from pathlib import Path\n", "import cattrs\n", "import json\n", - "from ipissa.train.train_adapter import proj_root, TrainingConfig\n" + "from antipasto.train.train_adapter import proj_root, TrainingConfig" ] }, { @@ -55,8 +57,7 @@ "# results_dir = Path(\"/workspace/InnerPiSSA_private/outputs/adapters/20251214_035340_g270m-antisym-r64-lr0.05\")\n", "results_dir = Path(\"../outputs/adapters/20260112_143322_q14b-antisym-r64-init1337/\")\n", "results_dir = Path(\"../outputs/adapters/20260112_112548_q4b-antisym-r64/\")\n", - "\n", - "/outputs/adapters/20260112_104520_olmo31-antisym-r64-init1337" + "# results_dir = Path(\"../outputs/adapters/20260112_104520_olmo31-antisym-r64-init1337\")" ] }, { @@ -77,7 +78,7 @@ "outputs": [], "source": [ "# Load adapter using new helper (replaces manual weight extraction + regexp building)\n", - "from ipissa.peft_utils.load import load_adapter\n", + "from antipasto.peft_utils.load import load_adapter\n", "\n", "model, tokenizer, layer_selection = load_adapter(results_dir, quantization_type=\"4bit\")\n", "print(f\"Loaded adapter from {results_dir}\")\n", @@ -99,8 +100,9 @@ "metadata": {}, "outputs": [], "source": [ - "from ipissa.gen import gen, gen_with_ans, gen_with_choices, get_choice_ids, ScaleAdapter\n", - "from ipissa.train.train_adapter import generate_example_output\n", + "from antipasto.gen import gen, gen_with_ans, gen_with_choices, ScaleAdapter\n", + "from antipasto.eval import get_choice_ids\n", + "from antipasto.train.train_adapter import generate_example_output\n", "choice_ids = get_choice_ids(tokenizer)" ] }, @@ -196,20 +198,22 @@ "cell_type": "code", "execution_count": null, "id": "c9123491", - "metadata": {}, + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [], "source": [ "\"\"\"\n", "Lets see how prompting + steering compares to just prompting\n", "\"\"\"\n", - "from ipissa.config import PROMPT, PERSONAS\n", + "from antipasto.config import PROMPT, PERSONAS\n", "s = \"You are late to work because you were out partying the night before. Do you tell your boss the truth or make up an excuse?\" + say_yn\n", "prompt1 = PROMPT.format(persona=PERSONAS[0][0]) + s\n", "prompt2 = PROMPT.format(persona=\"\") + s\n", "prompt3 = PROMPT.format(persona=PERSONAS[1][0]) + s\n", "list(gen_with_ans(model, tokenizer, prompt1, coeffs=[-100, -1, 0]))\n", "list(gen_with_ans(model, tokenizer, prompt2, coeffs=[-1, 0, 1]))\n", - "list(gen_with_ans(model, tokenizer, prompt3, coeffs=[0, 1, 100, 1000]));\n" + "list(gen_with_ans(model, tokenizer, prompt3, coeffs=[0, 1, 100, 1000]));" ] }, { diff --git a/nbs/talk_to_checkpoint.py b/nbs/talk_to_checkpoint.py index fc3703a..ee0bebc 100644 --- a/nbs/talk_to_checkpoint.py +++ b/nbs/talk_to_checkpoint.py @@ -22,7 +22,7 @@ import pandas as pd from pathlib import Path import cattrs import json -from ipissa.train.train_adapter import proj_root, TrainingConfig +from antipasto.train.train_adapter import proj_root, TrainingConfig # %% @@ -49,15 +49,14 @@ from ipissa.train.train_adapter import proj_root, TrainingConfig # results_dir = Path("/workspace/InnerPiSSA_private/outputs/adapters/20251214_035340_g270m-antisym-r64-lr0.05") results_dir = Path("../outputs/adapters/20260112_143322_q14b-antisym-r64-init1337/") results_dir = Path("../outputs/adapters/20260112_112548_q4b-antisym-r64/") - -/outputs/adapters/20260112_104520_olmo31-antisym-r64-init1337 +# results_dir = Path("../outputs/adapters/20260112_104520_olmo31-antisym-r64-init1337") # %% # !ls ../outputs/adapters/20260112_11* # %% # Load adapter using new helper (replaces manual weight extraction + regexp building) -from ipissa.peft_utils.load import load_adapter +from antipasto.peft_utils.load import load_adapter model, tokenizer, layer_selection = load_adapter(results_dir, quantization_type="4bit") print(f"Loaded adapter from {results_dir}") @@ -67,8 +66,9 @@ print(f"Layer selection: {len(layer_selection.adapter_layer_names)} adapter laye # ## Relicate # %% -from ipissa.gen import gen, gen_with_ans, gen_with_choices, get_choice_ids, ScaleAdapter -from ipissa.train.train_adapter import generate_example_output +from antipasto.gen import gen, gen_with_ans, gen_with_choices, ScaleAdapter +from antipasto.eval import get_choice_ids +from antipasto.train.train_adapter import generate_example_output choice_ids = get_choice_ids(tokenizer) # %% @@ -117,7 +117,7 @@ list(gen(model, tokenizer, prompt+say_yn)); """ Lets see how prompting + steering compares to just prompting """ -from ipissa.config import PROMPT, PERSONAS +from antipasto.config import PROMPT, PERSONAS s = "You are late to work because you were out partying the night before. Do you tell your boss the truth or make up an excuse?" + say_yn prompt1 = PROMPT.format(persona=PERSONAS[0][0]) + s prompt2 = PROMPT.format(persona="") + s diff --git a/scripts/upload_to_hf.py b/scripts/upload_to_hf.py new file mode 100644 index 0000000..dd632ab --- /dev/null +++ b/scripts/upload_to_hf.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Upload trained AntiPaSTO adapter to HuggingFace Hub. + +Usage: + uv run python scripts/upload_to_hf.py outputs/adapters/20260113_074001_g270m-antisym-r64 + uv run python scripts/upload_to_hf.py outputs/adapters/20260113_074001_g270m-antisym-r64 --repo-id wassname/antipasto-gemma-3-270m-honesty +""" +import json +import sys +from pathlib import Path +import tyro +from dataclasses import dataclass +from huggingface_hub import HfApi, create_repo +from loguru import logger + + +@dataclass +class UploadConfig: + adapter_path: Path + """Path to trained adapter folder""" + + repo_id: str = None + """HuggingFace repo ID (e.g., 'wassname/antipasto-gemma-3-1b-honesty'). Auto-generated if not provided.""" + + private: bool = False + """Make repo private""" + + +# Model name mappings for readable repo names +MODEL_SHORTCUTS = { + "g270m": "gemma-3-270m", + "g1b": "gemma-3-1b", + "g4b": "gemma-3-4b", + "q06b": "qwen2.5-0.6b", + "q4b": "qwen2.5-4b", + "q14b": "qwen2.5-14b", +} + + +def generate_repo_id(adapter_path: Path, username: str = "wassname") -> str: + """Generate HuggingFace repo ID from adapter folder name.""" + name = adapter_path.name + # Extract model shortcode from name like "20260113_074001_g270m-antisym-r64" + parts = name.split("_") + if len(parts) >= 3: + model_code = parts[2].split("-")[0] # e.g., "g270m" + model_name = MODEL_SHORTCUTS.get(model_code, model_code) + return f"{username}/antipasto-{model_name}-honesty" + return f"{username}/antipasto-adapter" + + +def create_model_card(adapter_path: Path, repo_id: str) -> str: + """Generate HuggingFace model card.""" + # Load training config if available + config_path = adapter_path / "training_config.json" + if config_path.exists(): + with open(config_path) as f: + training_config = json.load(f) + base_model = training_config.get("model_name", "unknown") + else: + base_model = "unknown" + + # Load adapter config + adapter_config_path = adapter_path / "adapter_config.json" + if adapter_config_path.exists(): + with open(adapter_config_path) as f: + adapter_config = json.load(f) + base_model = adapter_config.get("base_model_name_or_path", base_model) + rank = adapter_config.get("r", "unknown") + else: + rank = "unknown" + + return f'''--- +tags: + - antipasto + - peft + - moral-steering + - honesty + - alignment +base_model: {base_model} +library_name: peft +license: apache-2.0 +--- + +# AntiPaSTO: Honesty Steering Adapter + +[![arXiv](https://img.shields.io/badge/arXiv-2601.07473-b31b1b.svg)](https://arxiv.org/abs/2601.07473) + +🍝 **Anti-Pa**rallel **S**ubspace **T**raining for **O**rdered steering. + +This adapter steers language model responses toward honest or deceptive reasoning on moral dilemmas. + +## Usage + +```python +# Install +pip install git+https://github.com/wassname/AntiPaSTO.git + +from antipasto.peft_utils.load import load_adapter +from antipasto.gen import gen, ScaleAdapter + +# Load adapter +model, tokenizer, _ = load_adapter("{repo_id}", quantization_type="4bit") + +# Steer: coeff > 0 = honest, coeff < 0 = deceptive +prompt = "Should I tell my boss I was late because I overslept?" +with ScaleAdapter(model, coeff=1.0): + output = model.generate(**tokenizer(prompt, return_tensors="pt").to(model.device), max_new_tokens=64) + print(tokenizer.decode(output[0], skip_special_tokens=True)) +``` + +## Model Details + +- **Base model**: `{base_model}` +- **Adapter rank**: {rank} +- **Training data**: 800 synthetic honest/dishonest contrast pairs +- **Evaluation**: 1,360 Daily Dilemmas across 9 value dimensions + +## Citation + +```bibtex +@misc{{clark2026antipasto, + title = {{AntiPaSTO: Self-Supervised Steering of Moral Reasoning}}, + author = {{Clark, Michael J.}}, + year = {{2026}}, + eprint = {{2601.07473}}, + archivePrefix = {{arXiv}}, + primaryClass = {{cs.LG}}, + url = {{https://arxiv.org/abs/2601.07473}} +}} +``` +''' + + +def main(config: UploadConfig): + adapter_path = Path(config.adapter_path).resolve() + + if not adapter_path.exists(): + raise FileNotFoundError(f"Adapter path not found: {adapter_path}") + + # Check required files exist + required_files = [ + "adapter_config.json", + "adapter_model.safetensors", + "0_svd_bases.safetensors", + ] + for fname in required_files: + if not (adapter_path / fname).exists(): + raise FileNotFoundError(f"Required file missing: {adapter_path / fname}") + + # Generate repo_id if not provided + api = HfApi() + user_info = api.whoami() + username = user_info["name"] + + repo_id = config.repo_id or generate_repo_id(adapter_path, username) + logger.info(f"Uploading to: {repo_id}") + + # Create repo if needed + create_repo(repo_id, exist_ok=True, private=config.private) + + # Create model card + model_card = create_model_card(adapter_path, repo_id) + readme_path = adapter_path / "README.md" + with open(readme_path, "w") as f: + f.write(model_card) + + # Files to upload + files_to_upload = [ + "README.md", + "adapter_config.json", + "adapter_model.safetensors", + "0_svd_bases.safetensors", + "0_layer_selection.json", + "training_config.json", + ] + + for fname in files_to_upload: + fpath = adapter_path / fname + if fpath.exists(): + logger.info(f"Uploading: {fname}") + api.upload_file( + path_or_fileobj=str(fpath), + path_in_repo=fname, + repo_id=repo_id, + ) + + logger.info(f"✅ Uploaded to: https://huggingface.co/{repo_id}") + + +if __name__ == "__main__": + config = tyro.cli(UploadConfig) + main(config)