| { |
| "architecture": "FastDeepHybridLM", |
| "attention_every": 4, |
| "attention_residual_group_size": 4, |
| "bos_token_id": 1, |
| "conv_kernel_size": 4, |
| "embedding_size": 256, |
| "eos_token_id": 2, |
| "ffn_latent_size": 256, |
| "hidden_size": 512, |
| "initializer_range": 0.02, |
| "intermediate_size": 1728, |
| "max_position_embeddings": 4096, |
| "memory_heads": 4, |
| "memory_size": 128, |
| "model_type": "byte-deep-hybrid", |
| "mtp_loss_weight": 0.2, |
| "num_attention_heads": 8, |
| "num_hidden_layers": 24, |
| "num_key_value_heads": 2, |
| "pad_token_id": 0, |
| "padded_vocab_size": 6464, |
| "rms_norm_eps": 1e-05, |
| "rope_theta": 10000.0, |
| "vocab_size": 6422, |
| "window_size": 512 |
| } |
|
|
