mirror of
https://github.com/wassname/iris_bigvae.git
synced 2026-09-09 11:24:31 +08:00
10 lines
365 B
YAML
10 lines
365 B
YAML
_target_: src.models.TransformerConfig
|
|
max_blocks: 10 # this is the rollout length when training policy
|
|
num_layers: 1
|
|
num_heads: 1
|
|
embed_dim: 2048 # change this to whatever the embedding dimension is in your pretrained llm 2048 for llama. 2560 for stablelm
|
|
dropout: 0.1
|
|
model_name: "PY007/TinyLlama-1.1B-intermediate-step-715k-1.5T"
|
|
rank: 32
|
|
tokens_per_block: 17
|