mirror of
https://github.com/wassname/iris_bigvae.git
synced 2026-09-09 11:24:31 +08:00
wassname
328d281651
- use frozen transformer encoder where possible (obs, worldmodel, tokenizer, etc) - it's frozen - a huge vocab
9 lines
424 B
YAML
9 lines
424 B
YAML
_target_: src.models.TransformerConfig
|
|
max_blocks: 10 # this is the rollout length when training policy
|
|
tokens_per_block: 17 # how much info we can encode
|
|
dropout: 0.1
|
|
rank: 32 # lora rank
|
|
model_name: "PY007/TinyLlama-1.1B-intermediate-step-715k-1.5T"
|
|
vocab_size: 32000 # change to llm vocab dim
|
|
embed_dim: 2048 # change this to whatever the embedding dimension is in your pretrained llm 2048 for llama. 2560 for stablelm
|