pranavkarra
/

dense-5l-arxiv-code-simplestories

Text Generation

dense-transformer

text-generation-inference

Model card Files Files and versions

pranavkarra commited on Jul 14, 2025

Commit

7edc035

·

verified ·

1 Parent(s): 70a0587

Update config.json

Files changed (1) hide show

config.json +7 -5

config.json CHANGED Viewed

@@ -1,15 +1,17 @@
 {
-  "model_type": "tiny_gpt",
-  "architectures": ["TinyGPTLMHeadModel"],
   "vocab_size": 50256,
   "hidden_size": 768,
   "num_attention_heads": 12,
   "num_hidden_layers": 5,
-  "intermediate_size": 3072,
   "max_position_embeddings": 1024,
   "rms_norm_eps": 1e-6,
   "rope_theta": 10000.0,
   "attention_dropout": 0.1,
-  "hidden_dropout": 0.1,
-  "use_cache": true
 }

 {
+  "model_type": "llama",
+  "architectures": ["LlamaForCausalLM"],
   "vocab_size": 50256,
   "hidden_size": 768,
+  "intermediate_size": 3072,
   "num_attention_heads": 12,
   "num_hidden_layers": 5,
+  "num_key_value_heads": 12,
   "max_position_embeddings": 1024,
   "rms_norm_eps": 1e-6,
   "rope_theta": 10000.0,
   "attention_dropout": 0.1,
+  "mlp_bias": false,
+  "attention_bias": false,
+  "tie_word_embeddings": false
 }