mirror of
https://github.com/wassname/AntiPaSTO.git
synced 2026-09-11 11:52:42 +08:00
Make paper training presets explicit
Co-Authored-By: PI[openai-codex] <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
co-authored by
PI[openai-codex]
parent
5e0f8517f3
commit
6624f368b8
@@ -0,0 +1,41 @@
|
||||
from antipasto.config import default_configs
|
||||
from antipasto.train.train_adapter import is_optimizer_step, optimizer_steps_per_epoch
|
||||
|
||||
|
||||
def test_paper_v5_gemma1b_preset_matches_reported_hyperparameters():
|
||||
config = default_configs["paper-v5-gemma1b-24gb"][1]
|
||||
|
||||
assert config.model_name == "google/gemma-3-1b-it"
|
||||
assert config.seed == config.data_seed == 42
|
||||
assert config.max_samples == 800
|
||||
assert config.bs == 8
|
||||
assert config.effective_bs == 32
|
||||
assert config.n_epochs == 30
|
||||
assert config.lr == 1e-3
|
||||
assert config.wd == 1e-5
|
||||
assert config.r == 128
|
||||
assert config.n_modules == 64
|
||||
assert config.warmup_pct == 0.3
|
||||
assert config.mono_warmup_frac == 0.5
|
||||
assert config.loss_subspace_rank == 8
|
||||
assert config.loss_layer_frac == 0.9
|
||||
assert config.min_adapter_layer_frac == 0.1
|
||||
|
||||
|
||||
def test_gemma270m_24gb_preset():
|
||||
config = default_configs["gemma270m-24gb"][1]
|
||||
|
||||
assert config.model_name == "google/gemma-3-270m-it"
|
||||
assert config.bs == 24
|
||||
|
||||
|
||||
def test_optimizer_steps_include_final_partial_accumulation():
|
||||
updates = [
|
||||
batch_idx
|
||||
for batch_idx in range(85)
|
||||
if is_optimizer_step(batch_idx, n_batches=85, grad_accum_steps=4)
|
||||
]
|
||||
|
||||
assert updates[-1] == 84
|
||||
assert len(updates) == optimizer_steps_per_epoch(n_batches=85, grad_accum_steps=4) == 22
|
||||
assert optimizer_steps_per_epoch(n_batches=84, grad_accum_steps=4) == 21
|
||||
Reference in New Issue
Block a user