mirror of
https://github.com/wassname/AntiPaSTO.git
synced 2026-09-24 13:00:41 +08:00
wassnameandPI[openai-codex]
6624f368b8
Co-Authored-By: PI[openai-codex] <288921227+claudypoo@users.noreply.github.com>
42 lines
1.4 KiB
Python
42 lines
1.4 KiB
Python
from antipasto.config import default_configs
|
|
from antipasto.train.train_adapter import is_optimizer_step, optimizer_steps_per_epoch
|
|
|
|
|
|
def test_paper_v5_gemma1b_preset_matches_reported_hyperparameters():
|
|
config = default_configs["paper-v5-gemma1b-24gb"][1]
|
|
|
|
assert config.model_name == "google/gemma-3-1b-it"
|
|
assert config.seed == config.data_seed == 42
|
|
assert config.max_samples == 800
|
|
assert config.bs == 8
|
|
assert config.effective_bs == 32
|
|
assert config.n_epochs == 30
|
|
assert config.lr == 1e-3
|
|
assert config.wd == 1e-5
|
|
assert config.r == 128
|
|
assert config.n_modules == 64
|
|
assert config.warmup_pct == 0.3
|
|
assert config.mono_warmup_frac == 0.5
|
|
assert config.loss_subspace_rank == 8
|
|
assert config.loss_layer_frac == 0.9
|
|
assert config.min_adapter_layer_frac == 0.1
|
|
|
|
|
|
def test_gemma270m_24gb_preset():
|
|
config = default_configs["gemma270m-24gb"][1]
|
|
|
|
assert config.model_name == "google/gemma-3-270m-it"
|
|
assert config.bs == 24
|
|
|
|
|
|
def test_optimizer_steps_include_final_partial_accumulation():
|
|
updates = [
|
|
batch_idx
|
|
for batch_idx in range(85)
|
|
if is_optimizer_step(batch_idx, n_batches=85, grad_accum_steps=4)
|
|
]
|
|
|
|
assert updates[-1] == 84
|
|
assert len(updates) == optimizer_steps_per_epoch(n_batches=85, grad_accum_steps=4) == 22
|
|
assert optimizer_steps_per_epoch(n_batches=84, grad_accum_steps=4) == 21
|