Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+evaluation/level1_top_confusions.png filter=lfs diff=lfs merge=lfs -text
+evaluation/level2_top_confusions.png filter=lfs diff=lfs merge=lfs -text
+evaluation/confusion_flows.png filter=lfs diff=lfs merge=lfs -text
+evaluation/precision_recall_scatter.png filter=lfs diff=lfs merge=lfs -text
+evaluation/precision_recall_zoom.png filter=lfs diff=lfs merge=lfs -text
+evaluation/relative_performance_ranking.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,85 @@
+---
+language:
+- vi
+tags:
+- intent-classification
+- vietnamese
+---
+
+# gemma-3-4b-intent-1128-v3-lr1e05-bs2
+
+Fine-tuned model for Vietnamese intent classification.
+
+## Evaluation Results
+
+**Dataset**: `allganize/viettel-intent-augmented-total-1128` (validation)  
+**Samples**: 2,467  
+**Evaluated**: 2025-11-28 08:34:43
+
+### 📊 Overall Metrics
+
+| Metric | Score |
+|--------|-------|
+| **Format Accuracy** | 100.00% |
+| **Level 1 Accuracy** | 99.27% |
+| **Level 1 F1 (macro)** | 0.9884 |
+| **Level 2 Accuracy** | 97.97% |
+| **Level 2 F1 (macro)** | 0.9751 |
+| **Combined Accuracy** | 97.93% |
+| **Combined F1 (macro)** | 0.9742 |
+
+![Overall Metrics](evaluation/overall_metrics.png)
+
+### 🎯 Hierarchical Breakdown
+
+| Category | Rate | Count |
+|----------|------|-------|
+| Both Correct | 97.93% | 2416 |
+| Level 1 Only | 1.34% | 33 |
+| Both Wrong | 0.69% | 17 |
+
+### 🔍 Detailed Analysis
+
+**Confusion Patterns:**  
+![Confusion Flows](evaluation/confusion_flows.png)
+
+**Precision-Recall Analysis:**  
+![Precision-Recall Scatter](evaluation/precision_recall_scatter.png)
+
+**Intent Performance Distribution:**  
+![Intent Issues](evaluation/intent_issue_distribution.png)
+
+**Top Confused Intents:**  
+- Level 1: [View Heatmap](evaluation/level1_top_confusions.png)
+- Level 2: [View Heatmap](evaluation/level2_top_confusions.png)
+
+
+### 🎯 High-Performance Analysis
+
+**Zoomed Precision-Recall (0.85-1.0):**  
+![Precision-Recall Zoom](evaluation/precision_recall_zoom.png)
+
+**Relative Performance Ranking (Bottom 20):**  
+![Relative Ranking](evaluation/relative_performance_ranking.png)
+
+
+
+### 🎯 Intents for Improvement
+
+**Level 1:**
+
+- `Chat` (P: 97.44%, R: 92.68%, F1: 95.00%)
+- `FindMyPhone` (P: 100.00%, R: 92.86%, F1: 96.30%)
+- `News` (P: 97.62%, R: 97.62%, F1: 97.62%)
+- `UserQuery` (P: 95.92%, R: 100.00%, F1: 97.92%)
+
+**Level 2:**
+
+- `DeactivateSetting` (P: 93.33%, R: 82.35%, F1: 87.50%)
+- `NavigateMove` (P: 92.86%, R: 86.67%, F1: 89.66%)
+- `SuggestContent` (P: 81.82%, R: 100.00%, F1: 90.00%)
+- `ChannelDown` (P: 93.33%, R: 87.50%, F1: 90.32%)
+- `StopRinging` (P: 83.33%, R: 100.00%, F1: 90.91%)
+
+
+---

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image_soft_token>": 262144
+}

chat_template.jinja

@@ -0,0 +1,63 @@
+
+{{ bos_token }}
+
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {%- set first_user_prefix = messages[0]['content'] + '\n\n' -%}
+    {%- else -%}
+        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' -%}
+    {%- endif -%}
+    {%- set loop_messages = messages[1:] -%}
+{%- else -%}
+    {%- set first_user_prefix = "" -%}
+    {%- set loop_messages = messages -%}
+{%- endif -%}
+
+{%- for message in loop_messages -%}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
+        {{- raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") -}}
+    {%- endif -%}
+
+    {%- set role = "model" if message['role'] == 'assistant' else message['role'] -%}
+
+    {{- '<start_of_turn>' ~ role ~ '\n' ~ (first_user_prefix if loop.first else "") -}}
+
+    {%- if message['role'] == 'assistant' -%}
+        {% generation %}
+        {%- if message['content'] is string -%}
+            {{- message['content'] -}}
+        {%- elif message['content'] is iterable -%}
+            {%- for item in message['content'] -%}
+                {%- if item['type'] == 'image' -%}
+                    {{- '<start_of_image>' -}}
+                {%- elif item['type'] == 'text' -%}
+                    {{- item['text'] -}}
+                {%- endif -%}
+            {%- endfor -%}
+        {%- else -%}
+            {{- raise_exception("Invalid content type") -}}
+        {%- endif -%}
+        {{- '<end_of_turn>\n' -}}
+        {% endgeneration %}
+    {%- else -%}
+        {%- if message['content'] is string -%}
+            {{- message['content'] -}}
+        {%- elif message['content'] is iterable -%}
+            {%- for item in message['content'] -%}
+                {%- if item['type'] == 'image' -%}
+                    {{- '<start_of_image>' -}}
+                {%- elif item['type'] == 'text' -%}
+                    {{- item['text'] -}}
+                {%- endif -%}
+            {%- endfor -%}
+        {%- else -%}
+            {{- raise_exception("Invalid content type") -}}
+        {%- endif -%}
+        {{- '<end_of_turn>\n' -}}
+    {%- endif -%}
+
+{%- endfor -%}
+
+{%- if add_generation_prompt -%}
+    {{- '<start_of_turn>model\n' -}}
+{%- endif -%}

config.json

@@ -0,0 +1,97 @@
+{
+  "architectures": [
+    "Gemma3ForConditionalGeneration"
+  ],
+  "boi_token_index": 255999,
+  "bos_token_id": 2,
+  "dtype": "bfloat16",
+  "eoi_token_index": 256000,
+  "eos_token_id": 106,
+  "image_token_index": 262144,
+  "initializer_range": 0.02,
+  "mm_tokens_per_image": 256,
+  "model_type": "gemma3",
+  "pad_token_id": 106,
+  "text_config": {
+    "_sliding_window_pattern": 6,
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_logit_softcapping": null,
+    "dtype": "bfloat16",
+    "final_logit_softcapping": null,
+    "head_dim": 256,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "hidden_size": 2560,
+    "initializer_range": 0.02,
+    "intermediate_size": 10240,
+    "layer_types": [
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "model_type": "gemma3_text",
+    "num_attention_heads": 8,
+    "num_hidden_layers": 34,
+    "num_key_value_heads": 4,
+    "query_pre_attn_scalar": 256,
+    "rms_norm_eps": 1e-06,
+    "rope_local_base_freq": 10000.0,
+    "rope_scaling": {
+      "factor": 8.0,
+      "rope_type": "linear"
+    },
+    "rope_theta": 1000000.0,
+    "sliding_window": 1024,
+    "use_bidirectional_attention": false,
+    "use_cache": true,
+    "vocab_size": 262208
+  },
+  "transformers_version": "4.57.3",
+  "vision_config": {
+    "attention_dropout": 0.0,
+    "dtype": "bfloat16",
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "image_size": 896,
+    "intermediate_size": 4304,
+    "layer_norm_eps": 1e-06,
+    "model_type": "siglip_vision_model",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 27,
+    "patch_size": 14,
+    "vision_use_head": false
+  }
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "do_sample": true,
+  "eos_token_id": [
+    106,
+    1,
+    106
+  ],
+  "pad_token_id": 106,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "4.57.3"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fe795cccd05897f231ba58b0e61cdd2b1d192c5d9f957ba94431e1fa4447128
+size 4961251752

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e88463423f1ca34bc30560344d6b10a1e0cfb0a4867bdf6c4cd30edeba48f0fe
+size 3639026128

model.safetensors.index.json

@@ -0,0 +1,891 @@
+{
+  "metadata": {
+    "total_parameters": 4300079472,
+    "total_size": 8600158944
+  },
+  "weight_map": {
+    "language_model.model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.pre_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.pre_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.pre_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.pre_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.14.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.15.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.15.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.15.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.16.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.16.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.16.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.16.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.16.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.17.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.17.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.26.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.26.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.26.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.26.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.27.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.27.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.27.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.27.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.28.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.28.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.28.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.28.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.29.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.29.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.29.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.29.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.30.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.30.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.30.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.30.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.31.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.31.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.31.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.31.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.32.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.32.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.32.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.32.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.33.post_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.33.pre_feedforward_layernorm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.33.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.33.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.pre_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.pre_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.post_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.pre_feedforward_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.norm.weight": "model-00002-of-00002.safetensors",
+    "multi_modal_projector.mm_input_projection_weight": "model-00001-of-00002.safetensors",
+    "multi_modal_projector.mm_soft_emb_norm.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.embeddings.patch_embedding.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.embeddings.patch_embedding.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.embeddings.position_embedding.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.post_layernorm.bias": "model-00001-of-00002.safetensors",
+    "vision_tower.vision_model.post_layernorm.weight": "model-00001-of-00002.safetensors"
+  }
+}

preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_pan_and_scan": null,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Gemma3ImageProcessor",
+  "image_seq_length": 256,
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "pan_and_scan_max_num_crops": null,
+  "pan_and_scan_min_crop_size": null,
+  "pan_and_scan_min_ratio_to_activate": null,
+  "processor_class": "Gemma3Processor",
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 896,
+    "width": 896
+  }
+}

processor_config.json

@@ -0,0 +1,4 @@
+{
+  "image_seq_length": 256,
+  "processor_class": "Gemma3Processor"
+}

special_tokens_map.json

@@ -0,0 +1,21 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": "<end_of_turn>",
+  "image_token": "<image_soft_token>",
+  "pad_token": "<end_of_turn>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
+size 33384568

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
+size 4689074

trainer_state.json

@@ -0,0 +1,2695 @@
+{
+  "best_global_step": 662,
+  "best_metric": 0.005529946647584438,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 662,
+  "global_step": 1323,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 0.3124125599861145,
+      "epoch": 0.0007558578987150416,
+      "grad_norm": 420.0,
+      "learning_rate": 0.0,
+      "loss": 1.7321,
+      "mean_token_accuracy": 0.9224806427955627,
+      "num_tokens": 27164.0,
+      "step": 1
+    },
+    {
+      "entropy": 0.31456032395362854,
+      "epoch": 0.003779289493575208,
+      "grad_norm": 322.0,
+      "learning_rate": 3.007518796992481e-07,
+      "loss": 1.7027,
+      "mean_token_accuracy": 0.9347696453332901,
+      "num_tokens": 137780.0,
+      "step": 5
+    },
+    {
+      "entropy": 0.31488672494888303,
+      "epoch": 0.007558578987150416,
+      "grad_norm": 384.0,
+      "learning_rate": 6.766917293233083e-07,
+      "loss": 1.5286,
+      "mean_token_accuracy": 0.9335291385650635,
+      "num_tokens": 274385.0,
+      "step": 10
+    },
+    {
+      "entropy": 0.3156801164150238,
+      "epoch": 0.011337868480725623,
+      "grad_norm": 330.0,
+      "learning_rate": 1.0526315789473685e-06,
+      "loss": 0.9665,
+      "mean_token_accuracy": 0.9378681778907776,
+      "num_tokens": 408513.0,
+      "step": 15
+    },
+    {
+      "entropy": 0.3204445898532867,
+      "epoch": 0.015117157974300832,
+      "grad_norm": 55.25,
+      "learning_rate": 1.4285714285714286e-06,
+      "loss": 0.243,
+      "mean_token_accuracy": 0.9738323450088501,
+      "num_tokens": 546284.0,
+      "step": 20
+    },
+    {
+      "entropy": 0.32781667709350587,
+      "epoch": 0.01889644746787604,
+      "grad_norm": 21.5,
+      "learning_rate": 1.8045112781954887e-06,
+      "loss": 0.2025,
+      "mean_token_accuracy": 0.9826048254966736,
+      "num_tokens": 681832.0,
+      "step": 25
+    },
+    {
+      "entropy": 0.33683774471282957,
+      "epoch": 0.022675736961451247,
+      "grad_norm": 15.0625,
+      "learning_rate": 2.180451127819549e-06,
+      "loss": 0.0655,
+      "mean_token_accuracy": 0.99116530418396,
+      "num_tokens": 818921.0,
+      "step": 30
+    },
+    {
+      "entropy": 0.3569560945034027,
+      "epoch": 0.026455026455026454,
+      "grad_norm": 71.0,
+      "learning_rate": 2.556390977443609e-06,
+      "loss": 0.1554,
+      "mean_token_accuracy": 0.9790175199508667,
+      "num_tokens": 956260.0,
+      "step": 35
+    },
+    {
+      "entropy": 0.38554110527038576,
+      "epoch": 0.030234315948601664,
+      "grad_norm": 24.375,
+      "learning_rate": 2.9323308270676694e-06,
+      "loss": 0.0749,
+      "mean_token_accuracy": 0.985888135433197,
+      "num_tokens": 1093085.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.42414658665657046,
+      "epoch": 0.034013605442176874,
+      "grad_norm": 30.375,
+      "learning_rate": 3.3082706766917295e-06,
+      "loss": 0.0599,
+      "mean_token_accuracy": 0.9864688992500306,
+      "num_tokens": 1230268.0,
+      "step": 45
+    },
+    {
+      "entropy": 0.4906178951263428,
+      "epoch": 0.03779289493575208,
+      "grad_norm": 15.25,
+      "learning_rate": 3.6842105263157896e-06,
+      "loss": 0.0667,
+      "mean_token_accuracy": 0.9870380163192749,
+      "num_tokens": 1365940.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.5692261815071106,
+      "epoch": 0.04157218442932729,
+      "grad_norm": 9.3125,
+      "learning_rate": 4.06015037593985e-06,
+      "loss": 0.0422,
+      "mean_token_accuracy": 0.9911088466644287,
+      "num_tokens": 1503442.0,
+      "step": 55
+    },
+    {
+      "entropy": 0.6098211169242859,
+      "epoch": 0.045351473922902494,
+      "grad_norm": 4.5,
+      "learning_rate": 4.43609022556391e-06,
+      "loss": 0.0374,
+      "mean_token_accuracy": 0.9921163320541382,
+      "num_tokens": 1642345.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.6400599360466004,
+      "epoch": 0.0491307634164777,
+      "grad_norm": 10.6875,
+      "learning_rate": 4.81203007518797e-06,
+      "loss": 0.0392,
+      "mean_token_accuracy": 0.9900148630142211,
+      "num_tokens": 1778117.0,
+      "step": 65
+    },
+    {
+      "entropy": 0.6656645774841309,
+      "epoch": 0.05291005291005291,
+      "grad_norm": 8.25,
+      "learning_rate": 5.187969924812031e-06,
+      "loss": 0.0335,
+      "mean_token_accuracy": 0.9904999375343323,
+      "num_tokens": 1913704.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.6820027470588684,
+      "epoch": 0.05668934240362812,
+      "grad_norm": 11.625,
+      "learning_rate": 5.56390977443609e-06,
+      "loss": 0.0386,
+      "mean_token_accuracy": 0.9905120015144349,
+      "num_tokens": 2051532.0,
+      "step": 75
+    },
+    {
+      "entropy": 0.6896930813789368,
+      "epoch": 0.06046863189720333,
+      "grad_norm": 10.375,
+      "learning_rate": 5.939849624060151e-06,
+      "loss": 0.0407,
+      "mean_token_accuracy": 0.987898075580597,
+      "num_tokens": 2186524.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.646740996837616,
+      "epoch": 0.06424792139077853,
+      "grad_norm": 6.84375,
+      "learning_rate": 6.31578947368421e-06,
+      "loss": 0.0223,
+      "mean_token_accuracy": 0.991657304763794,
+      "num_tokens": 2322781.0,
+      "step": 85
+    },
+    {
+      "entropy": 0.5873156905174255,
+      "epoch": 0.06802721088435375,
+      "grad_norm": 11.875,
+      "learning_rate": 6.691729323308271e-06,
+      "loss": 0.0287,
+      "mean_token_accuracy": 0.9932127594947815,
+      "num_tokens": 2459839.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.5771235466003418,
+      "epoch": 0.07180650037792895,
+      "grad_norm": 6.78125,
+      "learning_rate": 7.067669172932331e-06,
+      "loss": 0.0504,
+      "mean_token_accuracy": 0.987032687664032,
+      "num_tokens": 2597176.0,
+      "step": 95
+    },
+    {
+      "entropy": 0.6098615527153015,
+      "epoch": 0.07558578987150416,
+      "grad_norm": 10.0,
+      "learning_rate": 7.4436090225563915e-06,
+      "loss": 0.0328,
+      "mean_token_accuracy": 0.9890416979789733,
+      "num_tokens": 2734219.0,
+      "step": 100
+    },
+    {
+      "entropy": 0.6399690747261048,
+      "epoch": 0.07936507936507936,
+      "grad_norm": 10.75,
+      "learning_rate": 7.81954887218045e-06,
+      "loss": 0.0357,
+      "mean_token_accuracy": 0.9906492829322815,
+      "num_tokens": 2869035.0,
+      "step": 105
+    },
+    {
+      "entropy": 0.6518049478530884,
+      "epoch": 0.08314436885865457,
+      "grad_norm": 8.625,
+      "learning_rate": 8.195488721804512e-06,
+      "loss": 0.0107,
+      "mean_token_accuracy": 0.9964231610298157,
+      "num_tokens": 3005281.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.6230420708656311,
+      "epoch": 0.08692365835222977,
+      "grad_norm": 9.875,
+      "learning_rate": 8.571428571428571e-06,
+      "loss": 0.0228,
+      "mean_token_accuracy": 0.9922025680541993,
+      "num_tokens": 3141872.0,
+      "step": 115
+    },
+    {
+      "entropy": 0.6244900345802307,
+      "epoch": 0.09070294784580499,
+      "grad_norm": 0.62109375,
+      "learning_rate": 8.947368421052632e-06,
+      "loss": 0.0156,
+      "mean_token_accuracy": 0.9958398461341857,
+      "num_tokens": 3279542.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.6464813113212585,
+      "epoch": 0.0944822373393802,
+      "grad_norm": 3.359375,
+      "learning_rate": 9.323308270676693e-06,
+      "loss": 0.0251,
+      "mean_token_accuracy": 0.9926198482513428,
+      "num_tokens": 3415299.0,
+      "step": 125
+    },
+    {
+      "entropy": 0.6796218752861023,
+      "epoch": 0.0982615268329554,
+      "grad_norm": 11.6875,
+      "learning_rate": 9.699248120300752e-06,
+      "loss": 0.038,
+      "mean_token_accuracy": 0.9911927700042724,
+      "num_tokens": 3551106.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.6995040655136109,
+      "epoch": 0.10204081632653061,
+      "grad_norm": 15.4375,
+      "learning_rate": 9.99998257609161e-06,
+      "loss": 0.0277,
+      "mean_token_accuracy": 0.994271981716156,
+      "num_tokens": 3688660.0,
+      "step": 135
+    },
+    {
+      "entropy": 0.7067576289176941,
+      "epoch": 0.10582010582010581,
+      "grad_norm": 2.484375,
+      "learning_rate": 9.999372752048729e-06,
+      "loss": 0.0115,
+      "mean_token_accuracy": 0.9963347554206848,
+      "num_tokens": 3825289.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.6457632541656494,
+      "epoch": 0.10959939531368103,
+      "grad_norm": 10.8125,
+      "learning_rate": 9.997891854019538e-06,
+      "loss": 0.0214,
+      "mean_token_accuracy": 0.9937294125556946,
+      "num_tokens": 3962011.0,
+      "step": 145
+    },
+    {
+      "entropy": 0.5923184990882874,
+      "epoch": 0.11337868480725624,
+      "grad_norm": 5.9375,
+      "learning_rate": 9.995540140030759e-06,
+      "loss": 0.0438,
+      "mean_token_accuracy": 0.9906680107116699,
+      "num_tokens": 4097931.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.6753766417503357,
+      "epoch": 0.11715797430083144,
+      "grad_norm": 4.75,
+      "learning_rate": 9.992318019837171e-06,
+      "loss": 0.0341,
+      "mean_token_accuracy": 0.993306303024292,
+      "num_tokens": 4235447.0,
+      "step": 155
+    },
+    {
+      "entropy": 0.7623552441596985,
+      "epoch": 0.12093726379440665,
+      "grad_norm": 6.21875,
+      "learning_rate": 9.988226054850218e-06,
+      "loss": 0.0207,
+      "mean_token_accuracy": 0.9943189144134521,
+      "num_tokens": 4370167.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.7839436531066895,
+      "epoch": 0.12471655328798185,
+      "grad_norm": 5.71875,
+      "learning_rate": 9.983264958040194e-06,
+      "loss": 0.0175,
+      "mean_token_accuracy": 0.9942814111709595,
+      "num_tokens": 4506649.0,
+      "step": 165
+    },
+    {
+      "entropy": 0.806551742553711,
+      "epoch": 0.12849584278155707,
+      "grad_norm": 6.1875,
+      "learning_rate": 9.977435593812013e-06,
+      "loss": 0.0295,
+      "mean_token_accuracy": 0.9921065330505371,
+      "num_tokens": 4643769.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.7947233080863952,
+      "epoch": 0.13227513227513227,
+      "grad_norm": 10.1875,
+      "learning_rate": 9.970738977854597e-06,
+      "loss": 0.0202,
+      "mean_token_accuracy": 0.9931598544120789,
+      "num_tokens": 4781667.0,
+      "step": 175
+    },
+    {
+      "entropy": 0.778699517250061,
+      "epoch": 0.1360544217687075,
+      "grad_norm": 9.4375,
+      "learning_rate": 9.963176276963916e-06,
+      "loss": 0.0211,
+      "mean_token_accuracy": 0.99481680393219,
+      "num_tokens": 4919296.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.7494423031806946,
+      "epoch": 0.1398337112622827,
+      "grad_norm": 0.447265625,
+      "learning_rate": 9.954748808839675e-06,
+      "loss": 0.0135,
+      "mean_token_accuracy": 0.9973663449287414,
+      "num_tokens": 5054459.0,
+      "step": 185
+    },
+    {
+      "entropy": 0.7103082180023194,
+      "epoch": 0.1436130007558579,
+      "grad_norm": 0.50390625,
+      "learning_rate": 9.945458041855732e-06,
+      "loss": 0.0242,
+      "mean_token_accuracy": 0.9963336825370789,
+      "num_tokens": 5189912.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.6840783715248108,
+      "epoch": 0.1473922902494331,
+      "grad_norm": 3.328125,
+      "learning_rate": 9.935305594804247e-06,
+      "loss": 0.0136,
+      "mean_token_accuracy": 0.996890377998352,
+      "num_tokens": 5326831.0,
+      "step": 195
+    },
+    {
+      "entropy": 0.7042184591293335,
+      "epoch": 0.15117157974300832,
+      "grad_norm": 0.875,
+      "learning_rate": 9.924293236613643e-06,
+      "loss": 0.0187,
+      "mean_token_accuracy": 0.9957641124725342,
+      "num_tokens": 5463018.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.7395557761192322,
+      "epoch": 0.15495086923658352,
+      "grad_norm": 2.453125,
+      "learning_rate": 9.91242288604037e-06,
+      "loss": 0.0175,
+      "mean_token_accuracy": 0.9968855857849122,
+      "num_tokens": 5600264.0,
+      "step": 205
+    },
+    {
+      "entropy": 0.7494800806045532,
+      "epoch": 0.15873015873015872,
+      "grad_norm": 1.0390625,
+      "learning_rate": 9.899696611334612e-06,
+      "loss": 0.0301,
+      "mean_token_accuracy": 0.994823443889618,
+      "num_tokens": 5736124.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.6967356204986572,
+      "epoch": 0.16250944822373395,
+      "grad_norm": 3.3125,
+      "learning_rate": 9.886116629879906e-06,
+      "loss": 0.0097,
+      "mean_token_accuracy": 0.9979151964187623,
+      "num_tokens": 5870255.0,
+      "step": 215
+    },
+    {
+      "entropy": 0.6044986009597778,
+      "epoch": 0.16628873771730915,
+      "grad_norm": 5.53125,
+      "learning_rate": 9.871685307806796e-06,
+      "loss": 0.0122,
+      "mean_token_accuracy": 0.9984508991241455,
+      "num_tokens": 6005956.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.5804512858390808,
+      "epoch": 0.17006802721088435,
+      "grad_norm": 7.34375,
+      "learning_rate": 9.85640515958057e-06,
+      "loss": 0.0215,
+      "mean_token_accuracy": 0.995808458328247,
+      "num_tokens": 6143559.0,
+      "step": 225
+    },
+    {
+      "entropy": 0.5928897976875305,
+      "epoch": 0.17384731670445955,
+      "grad_norm": 9.6875,
+      "learning_rate": 9.840278847563147e-06,
+      "loss": 0.0279,
+      "mean_token_accuracy": 0.9937070727348327,
+      "num_tokens": 6279410.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.6215349555015564,
+      "epoch": 0.17762660619803478,
+      "grad_norm": 3.5,
+      "learning_rate": 9.823309181549194e-06,
+      "loss": 0.0253,
+      "mean_token_accuracy": 0.9947557568550109,
+      "num_tokens": 6415542.0,
+      "step": 235
+    },
+    {
+      "entropy": 0.6546105861663818,
+      "epoch": 0.18140589569160998,
+      "grad_norm": 1.3125,
+      "learning_rate": 9.805499118276555e-06,
+      "loss": 0.02,
+      "mean_token_accuracy": 0.9947941064834595,
+      "num_tokens": 6550334.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.6813999891281128,
+      "epoch": 0.18518518518518517,
+      "grad_norm": 4.5625,
+      "learning_rate": 9.786851760911084e-06,
+      "loss": 0.0123,
+      "mean_token_accuracy": 0.996846055984497,
+      "num_tokens": 6686963.0,
+      "step": 245
+    },
+    {
+      "entropy": 0.6820224285125732,
+      "epoch": 0.1889644746787604,
+      "grad_norm": 4.3125,
+      "learning_rate": 9.767370358505958e-06,
+      "loss": 0.0074,
+      "mean_token_accuracy": 0.9979071021080017,
+      "num_tokens": 6823233.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.6676241278648376,
+      "epoch": 0.1927437641723356,
+      "grad_norm": 1.109375,
+      "learning_rate": 9.747058305435566e-06,
+      "loss": 0.0061,
+      "mean_token_accuracy": 0.9984600782394409,
+      "num_tokens": 6959250.0,
+      "step": 255
+    },
+    {
+      "entropy": 0.6509259819984436,
+      "epoch": 0.1965230536659108,
+      "grad_norm": 12.0,
+      "learning_rate": 9.7259191408041e-06,
+      "loss": 0.0169,
+      "mean_token_accuracy": 0.9963755011558533,
+      "num_tokens": 7098179.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.6398890137672424,
+      "epoch": 0.20030234315948603,
+      "grad_norm": 0.11181640625,
+      "learning_rate": 9.703956547828893e-06,
+      "loss": 0.0201,
+      "mean_token_accuracy": 0.9968681931495667,
+      "num_tokens": 7234547.0,
+      "step": 265
+    },
+    {
+      "entropy": 0.6418223738670349,
+      "epoch": 0.20408163265306123,
+      "grad_norm": 0.294921875,
+      "learning_rate": 9.681174353198687e-06,
+      "loss": 0.0042,
+      "mean_token_accuracy": 0.9989552617073059,
+      "num_tokens": 7370194.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.6441201329231262,
+      "epoch": 0.20786092214663643,
+      "grad_norm": 11.5,
+      "learning_rate": 9.657576526406872e-06,
+      "loss": 0.0184,
+      "mean_token_accuracy": 0.9953326106071472,
+      "num_tokens": 7507836.0,
+      "step": 275
+    },
+    {
+      "entropy": 0.6464055299758911,
+      "epoch": 0.21164021164021163,
+      "grad_norm": 3.796875,
+      "learning_rate": 9.633167179059859e-06,
+      "loss": 0.0127,
+      "mean_token_accuracy": 0.9968541741371155,
+      "num_tokens": 7644279.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.6436957597732544,
+      "epoch": 0.21541950113378686,
+      "grad_norm": 3.75,
+      "learning_rate": 9.607950564160682e-06,
+      "loss": 0.0113,
+      "mean_token_accuracy": 0.9973994374275208,
+      "num_tokens": 7780875.0,
+      "step": 285
+    },
+    {
+      "entropy": 0.6544555306434632,
+      "epoch": 0.21919879062736206,
+      "grad_norm": 4.78125,
+      "learning_rate": 9.581931075367979e-06,
+      "loss": 0.0199,
+      "mean_token_accuracy": 0.9953313350677491,
+      "num_tokens": 7917606.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.6784600496292115,
+      "epoch": 0.22297808012093726,
+      "grad_norm": 1.8515625,
+      "learning_rate": 9.555113246230443e-06,
+      "loss": 0.0097,
+      "mean_token_accuracy": 0.9973721981048584,
+      "num_tokens": 8054140.0,
+      "step": 295
+    },
+    {
+      "entropy": 0.7043081760406494,
+      "epoch": 0.22675736961451248,
+      "grad_norm": 1.421875,
+      "learning_rate": 9.527501749396924e-06,
+      "loss": 0.0164,
+      "mean_token_accuracy": 0.9963916540145874,
+      "num_tokens": 8189168.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.6775729060173035,
+      "epoch": 0.23053665910808768,
+      "grad_norm": 0.77734375,
+      "learning_rate": 9.499101395802277e-06,
+      "loss": 0.0017,
+      "mean_token_accuracy": 0.9994764447212219,
+      "num_tokens": 8325521.0,
+      "step": 305
+    },
+    {
+      "entropy": 0.620274543762207,
+      "epoch": 0.23431594860166288,
+      "grad_norm": 6.59375,
+      "learning_rate": 9.469917133829114e-06,
+      "loss": 0.0039,
+      "mean_token_accuracy": 0.9984441757202148,
+      "num_tokens": 8460969.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.5944037318229676,
+      "epoch": 0.23809523809523808,
+      "grad_norm": 2.734375,
+      "learning_rate": 9.439954048445628e-06,
+      "loss": 0.0158,
+      "mean_token_accuracy": 0.9974146604537963,
+      "num_tokens": 8598326.0,
+      "step": 315
+    },
+    {
+      "entropy": 0.5941187143325806,
+      "epoch": 0.2418745275888133,
+      "grad_norm": 5.0625,
+      "learning_rate": 9.409217360319594e-06,
+      "loss": 0.0145,
+      "mean_token_accuracy": 0.9974010944366455,
+      "num_tokens": 8734691.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.6153214812278748,
+      "epoch": 0.2456538170823885,
+      "grad_norm": 3.796875,
+      "learning_rate": 9.377712424908743e-06,
+      "loss": 0.0205,
+      "mean_token_accuracy": 0.9963401436805726,
+      "num_tokens": 8871429.0,
+      "step": 325
+    },
+    {
+      "entropy": 0.6333611607551575,
+      "epoch": 0.2494331065759637,
+      "grad_norm": 2.59375,
+      "learning_rate": 9.345444731527642e-06,
+      "loss": 0.0192,
+      "mean_token_accuracy": 0.9963601469993592,
+      "num_tokens": 9006202.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.6577699303627014,
+      "epoch": 0.25321239606953894,
+      "grad_norm": 6.625,
+      "learning_rate": 9.312419902391256e-06,
+      "loss": 0.0231,
+      "mean_token_accuracy": 0.9957833290100098,
+      "num_tokens": 9140608.0,
+      "step": 335
+    },
+    {
+      "entropy": 0.6712118268013001,
+      "epoch": 0.25699168556311414,
+      "grad_norm": 5.59375,
+      "learning_rate": 9.278643691635352e-06,
+      "loss": 0.0213,
+      "mean_token_accuracy": 0.9947887420654297,
+      "num_tokens": 9276772.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.6831606507301331,
+      "epoch": 0.26077097505668934,
+      "grad_norm": 0.119140625,
+      "learning_rate": 9.244121984313916e-06,
+      "loss": 0.0082,
+      "mean_token_accuracy": 0.9984793663024902,
+      "num_tokens": 9414540.0,
+      "step": 345
+    },
+    {
+      "entropy": 0.6921569228172302,
+      "epoch": 0.26455026455026454,
+      "grad_norm": 0.23046875,
+      "learning_rate": 9.208860795373765e-06,
+      "loss": 0.0119,
+      "mean_token_accuracy": 0.9963441491127014,
+      "num_tokens": 9550306.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.6849629878997803,
+      "epoch": 0.26832955404383974,
+      "grad_norm": 2.34375,
+      "learning_rate": 9.172866268606514e-06,
+      "loss": 0.0103,
+      "mean_token_accuracy": 0.9968430995941162,
+      "num_tokens": 9686568.0,
+      "step": 355
+    },
+    {
+      "entropy": 0.6642722845077514,
+      "epoch": 0.272108843537415,
+      "grad_norm": 7.34375,
+      "learning_rate": 9.136144675578114e-06,
+      "loss": 0.0193,
+      "mean_token_accuracy": 0.996327793598175,
+      "num_tokens": 9822879.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.6514992356300354,
+      "epoch": 0.2758881330309902,
+      "grad_norm": 5.78125,
+      "learning_rate": 9.098702414536107e-06,
+      "loss": 0.0089,
+      "mean_token_accuracy": 0.9973956227302552,
+      "num_tokens": 9959308.0,
+      "step": 365
+    },
+    {
+      "entropy": 0.6542440056800842,
+      "epoch": 0.2796674225245654,
+      "grad_norm": 1.9375,
+      "learning_rate": 9.060546009294818e-06,
+      "loss": 0.0127,
+      "mean_token_accuracy": 0.9979299783706665,
+      "num_tokens": 10096575.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.6637985825538635,
+      "epoch": 0.2834467120181406,
+      "grad_norm": 6.90625,
+      "learning_rate": 9.021682108098671e-06,
+      "loss": 0.0106,
+      "mean_token_accuracy": 0.9973872303962708,
+      "num_tokens": 10232621.0,
+      "step": 375
+    },
+    {
+      "entropy": 0.6581239342689514,
+      "epoch": 0.2872260015117158,
+      "grad_norm": 2.578125,
+      "learning_rate": 8.982117482463817e-06,
+      "loss": 0.0216,
+      "mean_token_accuracy": 0.9948861002922058,
+      "num_tokens": 10368276.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.6503197908401489,
+      "epoch": 0.291005291005291,
+      "grad_norm": 4.0625,
+      "learning_rate": 8.94185902599829e-06,
+      "loss": 0.0108,
+      "mean_token_accuracy": 0.9968842387199401,
+      "num_tokens": 10506811.0,
+      "step": 385
+    },
+    {
+      "entropy": 0.6437017798423768,
+      "epoch": 0.2947845804988662,
+      "grad_norm": 0.3125,
+      "learning_rate": 8.900913753200887e-06,
+      "loss": 0.0044,
+      "mean_token_accuracy": 0.9989569306373596,
+      "num_tokens": 10642953.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.6417622208595276,
+      "epoch": 0.29856386999244144,
+      "grad_norm": 0.014892578125,
+      "learning_rate": 8.859288798238988e-06,
+      "loss": 0.0098,
+      "mean_token_accuracy": 0.9979151129722595,
+      "num_tokens": 10780398.0,
+      "step": 395
+    },
+    {
+      "entropy": 0.6507818102836609,
+      "epoch": 0.30234315948601664,
+      "grad_norm": 2.921875,
+      "learning_rate": 8.816991413705515e-06,
+      "loss": 0.0225,
+      "mean_token_accuracy": 0.9968856692314148,
+      "num_tokens": 10916779.0,
+      "step": 400
+    },
+    {
+      "entropy": 0.6669408321380615,
+      "epoch": 0.30612244897959184,
+      "grad_norm": 4.8125,
+      "learning_rate": 8.774028969355273e-06,
+      "loss": 0.0171,
+      "mean_token_accuracy": 0.997943377494812,
+      "num_tokens": 11054380.0,
+      "step": 405
+    },
+    {
+      "entropy": 0.6745324969291687,
+      "epoch": 0.30990173847316704,
+      "grad_norm": 4.5,
+      "learning_rate": 8.730408950820864e-06,
+      "loss": 0.0142,
+      "mean_token_accuracy": 0.9974199891090393,
+      "num_tokens": 11191785.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.6699419260025025,
+      "epoch": 0.31368102796674224,
+      "grad_norm": 8.3125,
+      "learning_rate": 8.686138958308415e-06,
+      "loss": 0.0095,
+      "mean_token_accuracy": 0.9984320163726806,
+      "num_tokens": 11329447.0,
+      "step": 415
+    },
+    {
+      "entropy": 0.6696412801742554,
+      "epoch": 0.31746031746031744,
+      "grad_norm": 3.59375,
+      "learning_rate": 8.641226705273344e-06,
+      "loss": 0.0114,
+      "mean_token_accuracy": 0.9963794231414795,
+      "num_tokens": 11464507.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.6751126527786255,
+      "epoch": 0.32123960695389264,
+      "grad_norm": 2.671875,
+      "learning_rate": 8.595680017076403e-06,
+      "loss": 0.0126,
+      "mean_token_accuracy": 0.9973998427391052,
+      "num_tokens": 11599832.0,
+      "step": 425
+    },
+    {
+      "entropy": 0.6786884784698486,
+      "epoch": 0.3250188964474679,
+      "grad_norm": 0.34765625,
+      "learning_rate": 8.549506829620193e-06,
+      "loss": 0.001,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 11734763.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.6726262927055359,
+      "epoch": 0.3287981859410431,
+      "grad_norm": 1.6015625,
+      "learning_rate": 8.502715187966455e-06,
+      "loss": 0.0082,
+      "mean_token_accuracy": 0.9984387040138245,
+      "num_tokens": 11873050.0,
+      "step": 435
+    },
+    {
+      "entropy": 0.6710472464561462,
+      "epoch": 0.3325774754346183,
+      "grad_norm": 2.203125,
+      "learning_rate": 8.455313244934324e-06,
+      "loss": 0.0118,
+      "mean_token_accuracy": 0.9974292159080506,
+      "num_tokens": 12008849.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.6699627637863159,
+      "epoch": 0.3363567649281935,
+      "grad_norm": 4.625,
+      "learning_rate": 8.407309259679801e-06,
+      "loss": 0.0032,
+      "mean_token_accuracy": 0.9984238147735596,
+      "num_tokens": 12144651.0,
+      "step": 445
+    },
+    {
+      "entropy": 0.6549316048622131,
+      "epoch": 0.3401360544217687,
+      "grad_norm": 2.78125,
+      "learning_rate": 8.358711596256712e-06,
+      "loss": 0.0162,
+      "mean_token_accuracy": 0.9953205347061157,
+      "num_tokens": 12281801.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.6543108463287354,
+      "epoch": 0.3439153439153439,
+      "grad_norm": 0.078125,
+      "learning_rate": 8.309528722159383e-06,
+      "loss": 0.0068,
+      "mean_token_accuracy": 0.9994680881500244,
+      "num_tokens": 12418753.0,
+      "step": 455
+    },
+    {
+      "entropy": 0.6507919788360595,
+      "epoch": 0.3476946334089191,
+      "grad_norm": 2.984375,
+      "learning_rate": 8.259769206847286e-06,
+      "loss": 0.0074,
+      "mean_token_accuracy": 0.9963487148284912,
+      "num_tokens": 12555884.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.6534615635871888,
+      "epoch": 0.35147392290249435,
+      "grad_norm": 2.703125,
+      "learning_rate": 8.209441720251934e-06,
+      "loss": 0.0078,
+      "mean_token_accuracy": 0.9984305024147033,
+      "num_tokens": 12692226.0,
+      "step": 465
+    },
+    {
+      "entropy": 0.6645334720611572,
+      "epoch": 0.35525321239606955,
+      "grad_norm": 6.21875,
+      "learning_rate": 8.158555031266255e-06,
+      "loss": 0.021,
+      "mean_token_accuracy": 0.9947924494743348,
+      "num_tokens": 12829578.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.6773025989532471,
+      "epoch": 0.35903250188964475,
+      "grad_norm": 20.875,
+      "learning_rate": 8.107118006216732e-06,
+      "loss": 0.0171,
+      "mean_token_accuracy": 0.9963470816612243,
+      "num_tokens": 12967378.0,
+      "step": 475
+    },
+    {
+      "entropy": 0.6804514646530151,
+      "epoch": 0.36281179138321995,
+      "grad_norm": 2.453125,
+      "learning_rate": 8.055139607318558e-06,
+      "loss": 0.0099,
+      "mean_token_accuracy": 0.9979258418083191,
+      "num_tokens": 13106566.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.673608911037445,
+      "epoch": 0.36659108087679515,
+      "grad_norm": 0.16796875,
+      "learning_rate": 8.002628891114104e-06,
+      "loss": 0.0021,
+      "mean_token_accuracy": 0.9994818687438964,
+      "num_tokens": 13241319.0,
+      "step": 485
+    },
+    {
+      "entropy": 0.6570681810379029,
+      "epoch": 0.37037037037037035,
+      "grad_norm": 2.765625,
+      "learning_rate": 7.949595006894917e-06,
+      "loss": 0.0105,
+      "mean_token_accuracy": 0.9979082107543945,
+      "num_tokens": 13378007.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.6474269986152649,
+      "epoch": 0.3741496598639456,
+      "grad_norm": 1.71875,
+      "learning_rate": 7.896047195107599e-06,
+      "loss": 0.018,
+      "mean_token_accuracy": 0.9958997368812561,
+      "num_tokens": 13514007.0,
+      "step": 495
+    },
+    {
+      "entropy": 0.6409961104393005,
+      "epoch": 0.3779289493575208,
+      "grad_norm": 2.828125,
+      "learning_rate": 7.841994785743765e-06,
+      "loss": 0.0116,
+      "mean_token_accuracy": 0.9973924398422241,
+      "num_tokens": 13650732.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.6309547066688538,
+      "epoch": 0.381708238851096,
+      "grad_norm": 0.32421875,
+      "learning_rate": 7.787447196714428e-06,
+      "loss": 0.0063,
+      "mean_token_accuracy": 0.9984575271606445,
+      "num_tokens": 13788024.0,
+      "step": 505
+    },
+    {
+      "entropy": 0.6259261012077332,
+      "epoch": 0.3854875283446712,
+      "grad_norm": 0.03759765625,
+      "learning_rate": 7.73241393220905e-06,
+      "loss": 0.0061,
+      "mean_token_accuracy": 0.9979136824607849,
+      "num_tokens": 13924676.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.6200960636138916,
+      "epoch": 0.3892668178382464,
+      "grad_norm": 0.015869140625,
+      "learning_rate": 7.676904581039559e-06,
+      "loss": 0.0068,
+      "mean_token_accuracy": 0.9984575867652893,
+      "num_tokens": 14061789.0,
+      "step": 515
+    },
+    {
+      "entropy": 0.6154240965843201,
+      "epoch": 0.3930461073318216,
+      "grad_norm": 0.0255126953125,
+      "learning_rate": 7.620928814969636e-06,
+      "loss": 0.0131,
+      "mean_token_accuracy": 0.9969032168388366,
+      "num_tokens": 14198995.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.6134608268737793,
+      "epoch": 0.3968253968253968,
+      "grad_norm": 0.01055908203125,
+      "learning_rate": 7.564496387029532e-06,
+      "loss": 0.0082,
+      "mean_token_accuracy": 0.998963737487793,
+      "num_tokens": 14335595.0,
+      "step": 525
+    },
+    {
+      "entropy": 0.6157876014709472,
+      "epoch": 0.40060468631897206,
+      "grad_norm": 0.65234375,
+      "learning_rate": 7.507617129816733e-06,
+      "loss": 0.012,
+      "mean_token_accuracy": 0.9963508725166321,
+      "num_tokens": 14471802.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.6165508031845093,
+      "epoch": 0.40438397581254726,
+      "grad_norm": 0.01116943359375,
+      "learning_rate": 7.450300953782768e-06,
+      "loss": 0.0006,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 14607953.0,
+      "step": 535
+    },
+    {
+      "entropy": 0.6160001993179322,
+      "epoch": 0.40816326530612246,
+      "grad_norm": 0.041015625,
+      "learning_rate": 7.392557845506433e-06,
+      "loss": 0.0027,
+      "mean_token_accuracy": 0.9994897961616516,
+      "num_tokens": 14745750.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.6151047348976135,
+      "epoch": 0.41194255479969766,
+      "grad_norm": 9.4375,
+      "learning_rate": 7.3343978659537775e-06,
+      "loss": 0.0128,
+      "mean_token_accuracy": 0.9963156104087829,
+      "num_tokens": 14882036.0,
+      "step": 545
+    },
+    {
+      "entropy": 0.6159613847732544,
+      "epoch": 0.41572184429327286,
+      "grad_norm": 0.09814453125,
+      "learning_rate": 7.275831148725101e-06,
+      "loss": 0.0033,
+      "mean_token_accuracy": 0.9989569306373596,
+      "num_tokens": 15018906.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.6163604021072387,
+      "epoch": 0.41950113378684806,
+      "grad_norm": 0.042724609375,
+      "learning_rate": 7.216867898289319e-06,
+      "loss": 0.0014,
+      "mean_token_accuracy": 0.998973035812378,
+      "num_tokens": 15156568.0,
+      "step": 555
+    },
+    {
+      "entropy": 0.6148820281028747,
+      "epoch": 0.42328042328042326,
+      "grad_norm": 5.65625,
+      "learning_rate": 7.157518388205969e-06,
+      "loss": 0.0116,
+      "mean_token_accuracy": 0.9968932032585144,
+      "num_tokens": 15291333.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.6163430094718934,
+      "epoch": 0.4270597127739985,
+      "grad_norm": 0.052978515625,
+      "learning_rate": 7.097792959335169e-06,
+      "loss": 0.0082,
+      "mean_token_accuracy": 0.9979112029075623,
+      "num_tokens": 15430789.0,
+      "step": 565
+    },
+    {
+      "entropy": 0.6200400829315186,
+      "epoch": 0.4308390022675737,
+      "grad_norm": 0.349609375,
+      "learning_rate": 7.037702018035878e-06,
+      "loss": 0.0027,
+      "mean_token_accuracy": 0.9989610314369202,
+      "num_tokens": 15565867.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.620907187461853,
+      "epoch": 0.4346182917611489,
+      "grad_norm": 1.7421875,
+      "learning_rate": 6.977256034352713e-06,
+      "loss": 0.0038,
+      "mean_token_accuracy": 0.9994736790657044,
+      "num_tokens": 15700858.0,
+      "step": 575
+    },
+    {
+      "entropy": 0.6191686034202576,
+      "epoch": 0.4383975812547241,
+      "grad_norm": 0.1298828125,
+      "learning_rate": 6.916465540191692e-06,
+      "loss": 0.0069,
+      "mean_token_accuracy": 0.9984344959259033,
+      "num_tokens": 15835800.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.6172895193099975,
+      "epoch": 0.4421768707482993,
+      "grad_norm": 0.019287109375,
+      "learning_rate": 6.855341127485183e-06,
+      "loss": 0.0115,
+      "mean_token_accuracy": 0.9989583373069764,
+      "num_tokens": 15971026.0,
+      "step": 585
+    },
+    {
+      "entropy": 0.6186184287071228,
+      "epoch": 0.4459561602418745,
+      "grad_norm": 4.09375,
+      "learning_rate": 6.793893446346405e-06,
+      "loss": 0.0122,
+      "mean_token_accuracy": 0.9963900685310364,
+      "num_tokens": 16107078.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.623417603969574,
+      "epoch": 0.4497354497354497,
+      "grad_norm": 0.0634765625,
+      "learning_rate": 6.7321332032137875e-06,
+      "loss": 0.0008,
+      "mean_token_accuracy": 0.9994750618934631,
+      "num_tokens": 16242831.0,
+      "step": 595
+    },
+    {
+      "entropy": 0.6259370088577271,
+      "epoch": 0.45351473922902497,
+      "grad_norm": 0.134765625,
+      "learning_rate": 6.670071158985521e-06,
+      "loss": 0.0023,
+      "mean_token_accuracy": 0.998976981639862,
+      "num_tokens": 16382471.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.6233020544052124,
+      "epoch": 0.45729402872260017,
+      "grad_norm": 1.46875,
+      "learning_rate": 6.607718127144601e-06,
+      "loss": 0.0039,
+      "mean_token_accuracy": 0.9984204411506653,
+      "num_tokens": 16519391.0,
+      "step": 605
+    },
+    {
+      "entropy": 0.6223879218101501,
+      "epoch": 0.46107331821617537,
+      "grad_norm": 0.6328125,
+      "learning_rate": 6.545084971874738e-06,
+      "loss": 0.0068,
+      "mean_token_accuracy": 0.9979070544242858,
+      "num_tokens": 16654490.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.6202489614486695,
+      "epoch": 0.46485260770975056,
+      "grad_norm": 3.453125,
+      "learning_rate": 6.482182606167409e-06,
+      "loss": 0.0062,
+      "mean_token_accuracy": 0.997891879081726,
+      "num_tokens": 16792155.0,
+      "step": 615
+    },
+    {
+      "entropy": 0.619624662399292,
+      "epoch": 0.46863189720332576,
+      "grad_norm": 2.59375,
+      "learning_rate": 6.419021989920416e-06,
+      "loss": 0.0025,
+      "mean_token_accuracy": 0.9989583373069764,
+      "num_tokens": 16929384.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.6166529059410095,
+      "epoch": 0.47241118669690096,
+      "grad_norm": 1.9453125,
+      "learning_rate": 6.355614128028277e-06,
+      "loss": 0.0104,
+      "mean_token_accuracy": 0.99738609790802,
+      "num_tokens": 17065893.0,
+      "step": 625
+    },
+    {
+      "entropy": 0.6209856390953064,
+      "epoch": 0.47619047619047616,
+      "grad_norm": 1.8203125,
+      "learning_rate": 6.291970068464755e-06,
+      "loss": 0.0086,
+      "mean_token_accuracy": 0.998442780971527,
+      "num_tokens": 17201908.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.6346417427062988,
+      "epoch": 0.4799697656840514,
+      "grad_norm": 0.1435546875,
+      "learning_rate": 6.228100900357914e-06,
+      "loss": 0.0098,
+      "mean_token_accuracy": 0.9994791626930237,
+      "num_tokens": 17338449.0,
+      "step": 635
+    },
+    {
+      "entropy": 0.6447597503662109,
+      "epoch": 0.4837490551776266,
+      "grad_norm": 0.59375,
+      "learning_rate": 6.164017752057972e-06,
+      "loss": 0.0023,
+      "mean_token_accuracy": 0.9994764447212219,
+      "num_tokens": 17474392.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.6463032722473144,
+      "epoch": 0.4875283446712018,
+      "grad_norm": 2.375,
+      "learning_rate": 6.099731789198344e-06,
+      "loss": 0.0069,
+      "mean_token_accuracy": 0.9979228138923645,
+      "num_tokens": 17610576.0,
+      "step": 645
+    },
+    {
+      "entropy": 0.6476381182670593,
+      "epoch": 0.491307634164777,
+      "grad_norm": 3.203125,
+      "learning_rate": 6.035254212750172e-06,
+      "loss": 0.0031,
+      "mean_token_accuracy": 0.9994845390319824,
+      "num_tokens": 17748320.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.6472541451454162,
+      "epoch": 0.4950869236583522,
+      "grad_norm": 4.0,
+      "learning_rate": 5.970596257070711e-06,
+      "loss": 0.0076,
+      "mean_token_accuracy": 0.9984358310699463,
+      "num_tokens": 17884176.0,
+      "step": 655
+    },
+    {
+      "entropy": 0.6473153591156006,
+      "epoch": 0.4988662131519274,
+      "grad_norm": 0.318359375,
+      "learning_rate": 5.905769187945889e-06,
+      "loss": 0.0036,
+      "mean_token_accuracy": 0.9989583134651184,
+      "num_tokens": 18022027.0,
+      "step": 660
+    },
+    {
+      "epoch": 0.5003779289493575,
+      "eval_entropy": 0.6450142338871956,
+      "eval_loss": 0.005529946647584438,
+      "eval_mean_token_accuracy": 0.9985507105787595,
+      "eval_num_tokens": 18076506.0,
+      "eval_runtime": 16.3105,
+      "eval_samples_per_second": 34.885,
+      "eval_steps_per_second": 4.414,
+      "step": 662
+    },
+    {
+      "entropy": 0.6454153895378113,
+      "epoch": 0.5026455026455027,
+      "grad_norm": 0.1201171875,
+      "learning_rate": 5.840784300627396e-06,
+      "loss": 0.0043,
+      "mean_token_accuracy": 0.9989359736442566,
+      "num_tokens": 18156884.0,
+      "step": 665
+    },
+    {
+      "entropy": 0.6470770359039306,
+      "epoch": 0.5064247921390779,
+      "grad_norm": 8.0,
+      "learning_rate": 5.775652917864633e-06,
+      "loss": 0.0146,
+      "mean_token_accuracy": 0.9974155783653259,
+      "num_tokens": 18292386.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.6470168590545654,
+      "epoch": 0.5102040816326531,
+      "grad_norm": 0.0888671875,
+      "learning_rate": 5.710386387931886e-06,
+      "loss": 0.0022,
+      "mean_token_accuracy": 0.9994736790657044,
+      "num_tokens": 18428457.0,
+      "step": 675
+    },
+    {
+      "entropy": 0.640817391872406,
+      "epoch": 0.5139833711262283,
+      "grad_norm": 0.072265625,
+      "learning_rate": 5.644996082651018e-06,
+      "loss": 0.0006,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 18567506.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.6382765650749207,
+      "epoch": 0.5177626606198035,
+      "grad_norm": 0.0576171875,
+      "learning_rate": 5.579493395410105e-06,
+      "loss": 0.0053,
+      "mean_token_accuracy": 0.9989583253860473,
+      "num_tokens": 18704091.0,
+      "step": 685
+    },
+    {
+      "entropy": 0.6377130270004272,
+      "epoch": 0.5215419501133787,
+      "grad_norm": 6.21875,
+      "learning_rate": 5.51388973917828e-06,
+      "loss": 0.0115,
+      "mean_token_accuracy": 0.9979159593582153,
+      "num_tokens": 18839539.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.6364631175994873,
+      "epoch": 0.5253212396069539,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 5.448196544517168e-06,
+      "loss": 0.0005,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 18978610.0,
+      "step": 695
+    },
+    {
+      "entropy": 0.6324193835258484,
+      "epoch": 0.5291005291005291,
+      "grad_norm": 2.09375,
+      "learning_rate": 5.382425257589277e-06,
+      "loss": 0.0069,
+      "mean_token_accuracy": 0.9984482288360595,
+      "num_tokens": 19117618.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.6341057658195496,
+      "epoch": 0.5328798185941043,
+      "grad_norm": 1.75,
+      "learning_rate": 5.316587338163649e-06,
+      "loss": 0.0088,
+      "mean_token_accuracy": 0.997915244102478,
+      "num_tokens": 19255561.0,
+      "step": 705
+    },
+    {
+      "entropy": 0.6376821398735046,
+      "epoch": 0.5366591080876795,
+      "grad_norm": 0.055419921875,
+      "learning_rate": 5.2506942576191466e-06,
+      "loss": 0.0036,
+      "mean_token_accuracy": 0.9984469294548035,
+      "num_tokens": 19391688.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.6377916216850281,
+      "epoch": 0.5404383975812547,
+      "grad_norm": 1.9375,
+      "learning_rate": 5.184757496945726e-06,
+      "loss": 0.0067,
+      "mean_token_accuracy": 0.9989583253860473,
+      "num_tokens": 19528671.0,
+      "step": 715
+    },
+    {
+      "entropy": 0.641215717792511,
+      "epoch": 0.54421768707483,
+      "grad_norm": 0.0322265625,
+      "learning_rate": 5.118788544744016e-06,
+      "loss": 0.0021,
+      "mean_token_accuracy": 0.9989583253860473,
+      "num_tokens": 19665410.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.6453086376190186,
+      "epoch": 0.5479969765684052,
+      "grad_norm": 5.71875,
+      "learning_rate": 5.052798895223597e-06,
+      "loss": 0.0059,
+      "mean_token_accuracy": 0.9984346270561218,
+      "num_tokens": 19803509.0,
+      "step": 725
+    },
+    {
+      "entropy": 0.6470367074012756,
+      "epoch": 0.5517762660619804,
+      "grad_norm": 2.71875,
+      "learning_rate": 4.986800046200278e-06,
+      "loss": 0.0038,
+      "mean_token_accuracy": 0.9989582180976868,
+      "num_tokens": 19941001.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.6464150190353394,
+      "epoch": 0.5555555555555556,
+      "grad_norm": 0.044189453125,
+      "learning_rate": 4.920803497092757e-06,
+      "loss": 0.0017,
+      "mean_token_accuracy": 0.9994805216789245,
+      "num_tokens": 20076889.0,
+      "step": 735
+    },
+    {
+      "entropy": 0.6467163801193238,
+      "epoch": 0.5593348450491308,
+      "grad_norm": 5.4375,
+      "learning_rate": 4.854820746919005e-06,
+      "loss": 0.0069,
+      "mean_token_accuracy": 0.9968555450439454,
+      "num_tokens": 20212064.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.6457946181297303,
+      "epoch": 0.563114134542706,
+      "grad_norm": 5.4375,
+      "learning_rate": 4.78886329229271e-06,
+      "loss": 0.0099,
+      "mean_token_accuracy": 0.9974038600921631,
+      "num_tokens": 20348078.0,
+      "step": 745
+    },
+    {
+      "entropy": 0.6479183673858643,
+      "epoch": 0.5668934240362812,
+      "grad_norm": 1.5390625,
+      "learning_rate": 4.7229426254201504e-06,
+      "loss": 0.0047,
+      "mean_token_accuracy": 0.9984535336494446,
+      "num_tokens": 20483248.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.6519662022590638,
+      "epoch": 0.5706727135298564,
+      "grad_norm": 0.1123046875,
+      "learning_rate": 4.657070232097832e-06,
+      "loss": 0.0005,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 20619605.0,
+      "step": 755
+    },
+    {
+      "entropy": 0.6520264983177185,
+      "epoch": 0.5744520030234316,
+      "grad_norm": 0.1875,
+      "learning_rate": 4.591257589711233e-06,
+      "loss": 0.0009,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 20756384.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.6485134243965149,
+      "epoch": 0.5782312925170068,
+      "grad_norm": 0.318359375,
+      "learning_rate": 4.525516165235034e-06,
+      "loss": 0.0041,
+      "mean_token_accuracy": 0.9994832038879394,
+      "num_tokens": 20892551.0,
+      "step": 765
+    },
+    {
+      "entropy": 0.6489724397659302,
+      "epoch": 0.582010582010582,
+      "grad_norm": 0.15234375,
+      "learning_rate": 4.45985741323514e-06,
+      "loss": 0.0002,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 21028519.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.644879674911499,
+      "epoch": 0.5857898715041572,
+      "grad_norm": 0.146484375,
+      "learning_rate": 4.394292773872882e-06,
+      "loss": 0.0092,
+      "mean_token_accuracy": 0.9979193449020386,
+      "num_tokens": 21165463.0,
+      "step": 775
+    },
+    {
+      "entropy": 0.6453619122505188,
+      "epoch": 0.5895691609977324,
+      "grad_norm": 0.07421875,
+      "learning_rate": 4.3288336709117246e-06,
+      "loss": 0.0019,
+      "mean_token_accuracy": 0.9989473581314087,
+      "num_tokens": 21302633.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.6452991962432861,
+      "epoch": 0.5933484504913077,
+      "grad_norm": 0.6328125,
+      "learning_rate": 4.263491509726812e-06,
+      "loss": 0.0036,
+      "mean_token_accuracy": 0.9989500641822815,
+      "num_tokens": 21440200.0,
+      "step": 785
+    },
+    {
+      "entropy": 0.646388852596283,
+      "epoch": 0.5971277399848829,
+      "grad_norm": 0.06298828125,
+      "learning_rate": 4.198277675317758e-06,
+      "loss": 0.0021,
+      "mean_token_accuracy": 0.9994832038879394,
+      "num_tokens": 21575128.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.6419386029243469,
+      "epoch": 0.6009070294784581,
+      "grad_norm": 0.306640625,
+      "learning_rate": 4.133203530324938e-06,
+      "loss": 0.0028,
+      "mean_token_accuracy": 0.9994871735572814,
+      "num_tokens": 21712652.0,
+      "step": 795
+    },
+    {
+      "entropy": 0.6412655353546143,
+      "epoch": 0.6046863189720333,
+      "grad_norm": 1.0625,
+      "learning_rate": 4.068280413049715e-06,
+      "loss": 0.0025,
+      "mean_token_accuracy": 0.9989610433578491,
+      "num_tokens": 21849921.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.6409095525741577,
+      "epoch": 0.6084656084656085,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 4.003519635478889e-06,
+      "loss": 0.0034,
+      "mean_token_accuracy": 0.9994764447212219,
+      "num_tokens": 21987599.0,
+      "step": 805
+    },
+    {
+      "entropy": 0.6357825875282288,
+      "epoch": 0.6122448979591837,
+      "grad_norm": 0.400390625,
+      "learning_rate": 3.938932481313725e-06,
+      "loss": 0.0004,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 22125270.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.6361251711845398,
+      "epoch": 0.6160241874527589,
+      "grad_norm": 0.359375,
+      "learning_rate": 3.874530204003937e-06,
+      "loss": 0.0006,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 22259815.0,
+      "step": 815
+    },
+    {
+      "entropy": 0.6323129415512085,
+      "epoch": 0.6198034769463341,
+      "grad_norm": 1.0234375,
+      "learning_rate": 3.8103240247869077e-06,
+      "loss": 0.0051,
+      "mean_token_accuracy": 0.9979180097579956,
+      "num_tokens": 22395743.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.6306747913360595,
+      "epoch": 0.6235827664399093,
+      "grad_norm": 0.00927734375,
+      "learning_rate": 3.7463251307325432e-06,
+      "loss": 0.0032,
+      "mean_token_accuracy": 0.9994818687438964,
+      "num_tokens": 22531904.0,
+      "step": 825
+    },
+    {
+      "entropy": 0.6301920533180236,
+      "epoch": 0.6273620559334845,
+      "grad_norm": 0.07421875,
+      "learning_rate": 3.682544672794085e-06,
+      "loss": 0.0054,
+      "mean_token_accuracy": 0.998952841758728,
+      "num_tokens": 22668840.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.632958173751831,
+      "epoch": 0.6311413454270597,
+      "grad_norm": 0.859375,
+      "learning_rate": 3.618993763865185e-06,
+      "loss": 0.0024,
+      "mean_token_accuracy": 0.9994778037071228,
+      "num_tokens": 22805120.0,
+      "step": 835
+    },
+    {
+      "entropy": 0.6310474514961243,
+      "epoch": 0.6349206349206349,
+      "grad_norm": 0.1953125,
+      "learning_rate": 3.5556834768436498e-06,
+      "loss": 0.0004,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 22940501.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.6288277864456177,
+      "epoch": 0.6386999244142101,
+      "grad_norm": 0.05615234375,
+      "learning_rate": 3.4926248427021293e-06,
+      "loss": 0.0026,
+      "mean_token_accuracy": 0.9994910955429077,
+      "num_tokens": 23077572.0,
+      "step": 845
+    },
+    {
+      "entropy": 0.6270468473434448,
+      "epoch": 0.6424792139077853,
+      "grad_norm": 0.095703125,
+      "learning_rate": 3.4298288485661147e-06,
+      "loss": 0.0008,
+      "mean_token_accuracy": 0.9994778037071228,
+      "num_tokens": 23213864.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.6278021454811096,
+      "epoch": 0.6462585034013606,
+      "grad_norm": 2.734375,
+      "learning_rate": 3.3673064357995844e-06,
+      "loss": 0.0067,
+      "mean_token_accuracy": 0.9979260802268982,
+      "num_tokens": 23351632.0,
+      "step": 855
+    },
+    {
+      "entropy": 0.6229908108711243,
+      "epoch": 0.6500377928949358,
+      "grad_norm": 8.0,
+      "learning_rate": 3.3050684980986105e-06,
+      "loss": 0.003,
+      "mean_token_accuracy": 0.9984301805496216,
+      "num_tokens": 23486525.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.6223344326019287,
+      "epoch": 0.653817082388511,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 3.2431258795932863e-06,
+      "loss": 0.0024,
+      "mean_token_accuracy": 0.9984441995620728,
+      "num_tokens": 23624986.0,
+      "step": 865
+    },
+    {
+      "entropy": 0.6258201241493225,
+      "epoch": 0.6575963718820862,
+      "grad_norm": 3.03125,
+      "learning_rate": 3.181489372958272e-06,
+      "loss": 0.0042,
+      "mean_token_accuracy": 0.9989372611045837,
+      "num_tokens": 23762002.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.6200236201286315,
+      "epoch": 0.6613756613756614,
+      "grad_norm": 0.11767578125,
+      "learning_rate": 3.120169717532319e-06,
+      "loss": 0.0047,
+      "mean_token_accuracy": 0.9994845390319824,
+      "num_tokens": 23897588.0,
+      "step": 875
+    },
+    {
+      "entropy": 0.6229296207427979,
+      "epoch": 0.6651549508692366,
+      "grad_norm": 6.90625,
+      "learning_rate": 3.059177597447087e-06,
+      "loss": 0.0063,
+      "mean_token_accuracy": 0.9984667658805847,
+      "num_tokens": 24036641.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.6244211196899414,
+      "epoch": 0.6689342403628118,
+      "grad_norm": 0.029052734375,
+      "learning_rate": 2.9985236397655726e-06,
+      "loss": 0.0131,
+      "mean_token_accuracy": 0.9974199652671814,
+      "num_tokens": 24171544.0,
+      "step": 885
+    },
+    {
+      "entropy": 0.6283087849617004,
+      "epoch": 0.672713529856387,
+      "grad_norm": 0.054931640625,
+      "learning_rate": 2.9382184126304834e-06,
+      "loss": 0.0013,
+      "mean_token_accuracy": 0.9994858622550964,
+      "num_tokens": 24309060.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.6263309836387634,
+      "epoch": 0.6764928193499622,
+      "grad_norm": 0.00958251953125,
+      "learning_rate": 2.8782724234228876e-06,
+      "loss": 0.0029,
+      "mean_token_accuracy": 0.9989610075950622,
+      "num_tokens": 24444653.0,
+      "step": 895
+    },
+    {
+      "entropy": 0.6234323501586914,
+      "epoch": 0.6802721088435374,
+      "grad_norm": 0.05419921875,
+      "learning_rate": 2.818696116931431e-06,
+      "loss": 0.0005,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 24580885.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.6275152206420899,
+      "epoch": 0.6840513983371126,
+      "grad_norm": 0.00946044921875,
+      "learning_rate": 2.7594998735324905e-06,
+      "loss": 0.0016,
+      "mean_token_accuracy": 0.9994845390319824,
+      "num_tokens": 24719457.0,
+      "step": 905
+    },
+    {
+      "entropy": 0.6263895153999328,
+      "epoch": 0.6878306878306878,
+      "grad_norm": 0.06494140625,
+      "learning_rate": 2.7006940073815136e-06,
+      "loss": 0.0011,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 24856055.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.6252614855766296,
+      "epoch": 0.691609977324263,
+      "grad_norm": 1.0234375,
+      "learning_rate": 2.6422887646159234e-06,
+      "loss": 0.0063,
+      "mean_token_accuracy": 0.997401237487793,
+      "num_tokens": 24992445.0,
+      "step": 915
+    },
+    {
+      "entropy": 0.6215481519699096,
+      "epoch": 0.6953892668178382,
+      "grad_norm": 11.75,
+      "learning_rate": 2.584294321569862e-06,
+      "loss": 0.0035,
+      "mean_token_accuracy": 0.9989569306373596,
+      "num_tokens": 25128519.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.6217891812324524,
+      "epoch": 0.6991685563114135,
+      "grad_norm": 0.01300048828125,
+      "learning_rate": 2.526720783001107e-06,
+      "loss": 0.0012,
+      "mean_token_accuracy": 0.9994858622550964,
+      "num_tokens": 25266464.0,
+      "step": 925
+    },
+    {
+      "entropy": 0.6230118870735168,
+      "epoch": 0.7029478458049887,
+      "grad_norm": 0.068359375,
+      "learning_rate": 2.469578180330444e-06,
+      "loss": 0.0026,
+      "mean_token_accuracy": 0.998940396308899,
+      "num_tokens": 25402715.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.6251353025436401,
+      "epoch": 0.7067271352985639,
+      "grad_norm": 0.01531982421875,
+      "learning_rate": 2.4128764698938297e-06,
+      "loss": 0.0023,
+      "mean_token_accuracy": 0.9994764447212219,
+      "num_tokens": 25539042.0,
+      "step": 935
+    },
+    {
+      "entropy": 0.6235007047653198,
+      "epoch": 0.7105064247921391,
+      "grad_norm": 7.53125,
+      "learning_rate": 2.356625531207638e-06,
+      "loss": 0.0105,
+      "mean_token_accuracy": 0.9973794341087341,
+      "num_tokens": 25673951.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.6224774122238159,
+      "epoch": 0.7142857142857143,
+      "grad_norm": 0.5234375,
+      "learning_rate": 2.3008351652472714e-06,
+      "loss": 0.0029,
+      "mean_token_accuracy": 0.9989608526229858,
+      "num_tokens": 25809956.0,
+      "step": 945
+    },
+    {
+      "entropy": 0.6207262992858886,
+      "epoch": 0.7180650037792895,
+      "grad_norm": 3.8125,
+      "learning_rate": 2.245515092739488e-06,
+      "loss": 0.0041,
+      "mean_token_accuracy": 0.9984666347503662,
+      "num_tokens": 25946173.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.6216147780418396,
+      "epoch": 0.7218442932728647,
+      "grad_norm": 0.828125,
+      "learning_rate": 2.1906749524686856e-06,
+      "loss": 0.0046,
+      "mean_token_accuracy": 0.9994805216789245,
+      "num_tokens": 26082522.0,
+      "step": 955
+    },
+    {
+      "entropy": 0.6248481035232544,
+      "epoch": 0.7256235827664399,
+      "grad_norm": 0.09814453125,
+      "learning_rate": 2.136324299597474e-06,
+      "loss": 0.0068,
+      "mean_token_accuracy": 0.9989418029785156,
+      "num_tokens": 26220220.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.6239413142204284,
+      "epoch": 0.7294028722600151,
+      "grad_norm": 6.625,
+      "learning_rate": 2.0824726040018174e-06,
+      "loss": 0.005,
+      "mean_token_accuracy": 0.9984360456466674,
+      "num_tokens": 26356547.0,
+      "step": 965
+    },
+    {
+      "entropy": 0.6228319883346558,
+      "epoch": 0.7331821617535903,
+      "grad_norm": 0.03955078125,
+      "learning_rate": 2.0291292486210327e-06,
+      "loss": 0.0025,
+      "mean_token_accuracy": 0.9989743590354919,
+      "num_tokens": 26493643.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.6238638162612915,
+      "epoch": 0.7369614512471655,
+      "grad_norm": 0.04150390625,
+      "learning_rate": 1.976303527822933e-06,
+      "loss": 0.0022,
+      "mean_token_accuracy": 0.9994858622550964,
+      "num_tokens": 26630319.0,
+      "step": 975
+    },
+    {
+      "entropy": 0.6228212952613831,
+      "epoch": 0.7407407407407407,
+      "grad_norm": 0.041748046875,
+      "learning_rate": 1.9240046457844223e-06,
+      "loss": 0.0025,
+      "mean_token_accuracy": 0.9989431500434875,
+      "num_tokens": 26767214.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.6239274144172668,
+      "epoch": 0.7445200302343159,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.8722417148877752e-06,
+      "loss": 0.0053,
+      "mean_token_accuracy": 0.9989583253860473,
+      "num_tokens": 26903449.0,
+      "step": 985
+    },
+    {
+      "entropy": 0.6227865815162659,
+      "epoch": 0.7482993197278912,
+      "grad_norm": 0.046875,
+      "learning_rate": 1.821023754132933e-06,
+      "loss": 0.0081,
+      "mean_token_accuracy": 0.9974170565605164,
+      "num_tokens": 27041212.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.6259052991867066,
+      "epoch": 0.7520786092214664,
+      "grad_norm": 4.03125,
+      "learning_rate": 1.7703596875660645e-06,
+      "loss": 0.0097,
+      "mean_token_accuracy": 0.9979274153709412,
+      "num_tokens": 27177799.0,
+      "step": 995
+    },
+    {
+      "entropy": 0.6243377447128295,
+      "epoch": 0.7558578987150416,
+      "grad_norm": 5.25,
+      "learning_rate": 1.7202583427246633e-06,
+      "loss": 0.0073,
+      "mean_token_accuracy": 0.9968909025192261,
+      "num_tokens": 27315021.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.6245812058448792,
+      "epoch": 0.7596371882086168,
+      "grad_norm": 2.4375,
+      "learning_rate": 1.6707284490994746e-06,
+      "loss": 0.0078,
+      "mean_token_accuracy": 0.997928786277771,
+      "num_tokens": 27450263.0,
+      "step": 1005
+    },
+    {
+      "entropy": 0.6255474805831909,
+      "epoch": 0.763416477702192,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.6217786366134953e-06,
+      "loss": 0.0041,
+      "mean_token_accuracy": 0.9994791626930237,
+      "num_tokens": 27586816.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.6291083574295044,
+      "epoch": 0.7671957671957672,
+      "grad_norm": 0.05419921875,
+      "learning_rate": 1.5734174341183284e-06,
+      "loss": 0.0014,
+      "mean_token_accuracy": 0.9994750618934631,
+      "num_tokens": 27722646.0,
+      "step": 1015
+    },
+    {
+      "entropy": 0.6277247428894043,
+      "epoch": 0.7709750566893424,
+      "grad_norm": 0.25,
+      "learning_rate": 1.525653267908132e-06,
+      "loss": 0.0007,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 27860053.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.6258515357971192,
+      "epoch": 0.7747543461829176,
+      "grad_norm": 0.1376953125,
+      "learning_rate": 1.4784944602514662e-06,
+      "loss": 0.0039,
+      "mean_token_accuracy": 0.9989677429199219,
+      "num_tokens": 27998175.0,
+      "step": 1025
+    },
+    {
+      "entropy": 0.6294663429260254,
+      "epoch": 0.7785336356764928,
+      "grad_norm": 0.044677734375,
+      "learning_rate": 1.4319492279412388e-06,
+      "loss": 0.0014,
+      "mean_token_accuracy": 0.9994652390480041,
+      "num_tokens": 28132887.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.6290977954864502,
+      "epoch": 0.782312925170068,
+      "grad_norm": 5.3125,
+      "learning_rate": 1.3860256808630429e-06,
+      "loss": 0.002,
+      "mean_token_accuracy": 0.9994910955429077,
+      "num_tokens": 28268986.0,
+      "step": 1035
+    },
+    {
+      "entropy": 0.6268069744110107,
+      "epoch": 0.7860922146636432,
+      "grad_norm": 4.15625,
+      "learning_rate": 1.340731820582114e-06,
+      "loss": 0.0085,
+      "mean_token_accuracy": 0.9984454393386841,
+      "num_tokens": 28406196.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.6279515743255615,
+      "epoch": 0.7898715041572184,
+      "grad_norm": 0.12353515625,
+      "learning_rate": 1.2960755389491703e-06,
+      "loss": 0.0057,
+      "mean_token_accuracy": 0.9989556193351745,
+      "num_tokens": 28543139.0,
+      "step": 1045
+    },
+    {
+      "entropy": 0.6263696193695069,
+      "epoch": 0.7936507936507936,
+      "grad_norm": 0.07763671875,
+      "learning_rate": 1.2520646167253514e-06,
+      "loss": 0.0041,
+      "mean_token_accuracy": 0.9994845390319824,
+      "num_tokens": 28679153.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.6293219327926636,
+      "epoch": 0.7974300831443688,
+      "grad_norm": 0.12158203125,
+      "learning_rate": 1.2087067222265409e-06,
+      "loss": 0.0006,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 28815977.0,
+      "step": 1055
+    },
+    {
+      "entropy": 0.6283244490623474,
+      "epoch": 0.8012093726379441,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.166009409987251e-06,
+      "loss": 0.0008,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 28950890.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.6297204375267029,
+      "epoch": 0.8049886621315193,
+      "grad_norm": 0.86328125,
+      "learning_rate": 1.1239801194443507e-06,
+      "loss": 0.0039,
+      "mean_token_accuracy": 0.998441469669342,
+      "num_tokens": 29088081.0,
+      "step": 1065
+    },
+    {
+      "entropy": 0.6257842779159546,
+      "epoch": 0.8087679516250945,
+      "grad_norm": 1.453125,
+      "learning_rate": 1.082626173640846e-06,
+      "loss": 0.0072,
+      "mean_token_accuracy": 0.9989416837692261,
+      "num_tokens": 29227491.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.6270438313484192,
+      "epoch": 0.8125472411186697,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.0419547779499283e-06,
+      "loss": 0.0065,
+      "mean_token_accuracy": 0.9984452605247498,
+      "num_tokens": 29363388.0,
+      "step": 1075
+    },
+    {
+      "entropy": 0.625947892665863,
+      "epoch": 0.8163265306122449,
+      "grad_norm": 5.09375,
+      "learning_rate": 1.0019730188195464e-06,
+      "loss": 0.0055,
+      "mean_token_accuracy": 0.9984427571296692,
+      "num_tokens": 29498004.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.6268860936164856,
+      "epoch": 0.8201058201058201,
+      "grad_norm": 3.921875,
+      "learning_rate": 9.626878625376784e-07,
+      "loss": 0.0076,
+      "mean_token_accuracy": 0.9984326839447022,
+      "num_tokens": 29634999.0,
+      "step": 1085
+    },
+    {
+      "entropy": 0.6276703119277954,
+      "epoch": 0.8238851095993953,
+      "grad_norm": 0.0291748046875,
+      "learning_rate": 9.241061540185547e-07,
+      "loss": 0.0009,
+      "mean_token_accuracy": 0.9994778037071228,
+      "num_tokens": 29773857.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.6304038643836976,
+      "epoch": 0.8276643990929705,
+      "grad_norm": 0.0185546875,
+      "learning_rate": 8.862346156100188e-07,
+      "loss": 0.0021,
+      "mean_token_accuracy": 0.9989361643791199,
+      "num_tokens": 29910413.0,
+      "step": 1095
+    },
+    {
+      "entropy": 0.6274094462394715,
+      "epoch": 0.8314436885865457,
+      "grad_norm": 2.328125,
+      "learning_rate": 8.490798459222477e-07,
+      "loss": 0.0006,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 30047404.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.6275372743606568,
+      "epoch": 0.8352229780801209,
+      "grad_norm": 0.33984375,
+      "learning_rate": 8.12648318678036e-07,
+      "loss": 0.0061,
+      "mean_token_accuracy": 0.9984293103218078,
+      "num_tokens": 30183552.0,
+      "step": 1105
+    },
+    {
+      "entropy": 0.6304641485214233,
+      "epoch": 0.8390022675736961,
+      "grad_norm": 0.09228515625,
+      "learning_rate": 7.769463815848344e-07,
+      "loss": 0.0038,
+      "mean_token_accuracy": 0.9994736790657044,
+      "num_tokens": 30319430.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.6279330015182495,
+      "epoch": 0.8427815570672713,
+      "grad_norm": 10.1875,
+      "learning_rate": 7.419802552287453e-07,
+      "loss": 0.011,
+      "mean_token_accuracy": 0.9979191064834595,
+      "num_tokens": 30457112.0,
+      "step": 1115
+    },
+    {
+      "entropy": 0.6278607010841369,
+      "epoch": 0.8465608465608465,
+      "grad_norm": 0.73046875,
+      "learning_rate": 7.077560319906696e-07,
+      "loss": 0.0092,
+      "mean_token_accuracy": 0.9979125261306763,
+      "num_tokens": 30593910.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.6293578505516052,
+      "epoch": 0.8503401360544217,
+      "grad_norm": 0.427734375,
+      "learning_rate": 6.742796749847908e-07,
+      "loss": 0.0017,
+      "mean_token_accuracy": 0.9994884967803955,
+      "num_tokens": 30730370.0,
+      "step": 1125
+    },
+    {
+      "entropy": 0.6294125199317933,
+      "epoch": 0.854119425547997,
+      "grad_norm": 0.01611328125,
+      "learning_rate": 6.415570170195801e-07,
+      "loss": 0.0001,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 30864681.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.6303628087043762,
+      "epoch": 0.8578987150415722,
+      "grad_norm": 0.5859375,
+      "learning_rate": 6.095937595815104e-07,
+      "loss": 0.0005,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 30999618.0,
+      "step": 1135
+    },
+    {
+      "entropy": 0.6290588498115539,
+      "epoch": 0.8616780045351474,
+      "grad_norm": 20.875,
+      "learning_rate": 5.783954718416468e-07,
+      "loss": 0.012,
+      "mean_token_accuracy": 0.9984236717224121,
+      "num_tokens": 31136806.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.6272745132446289,
+      "epoch": 0.8654572940287226,
+      "grad_norm": 0.08447265625,
+      "learning_rate": 5.47967589685292e-07,
+      "loss": 0.0034,
+      "mean_token_accuracy": 0.9994764447212219,
+      "num_tokens": 31272561.0,
+      "step": 1145
+    },
+    {
+      "entropy": 0.6306499123573304,
+      "epoch": 0.8692365835222978,
+      "grad_norm": 3.265625,
+      "learning_rate": 5.183154147648578e-07,
+      "loss": 0.0006,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 31407553.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.6274000406265259,
+      "epoch": 0.873015873015873,
+      "grad_norm": 2.140625,
+      "learning_rate": 4.894441135761197e-07,
+      "loss": 0.0025,
+      "mean_token_accuracy": 0.998951506614685,
+      "num_tokens": 31543669.0,
+      "step": 1155
+    },
+    {
+      "entropy": 0.6274757504463195,
+      "epoch": 0.8767951625094482,
+      "grad_norm": 6.3125,
+      "learning_rate": 4.613587165580269e-07,
+      "loss": 0.0027,
+      "mean_token_accuracy": 0.998970365524292,
+      "num_tokens": 31679842.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.6302230358123779,
+      "epoch": 0.8805744520030234,
+      "grad_norm": 0.10107421875,
+      "learning_rate": 4.34064117216218e-07,
+      "loss": 0.0002,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 31817091.0,
+      "step": 1165
+    },
+    {
+      "entropy": 0.6279751658439636,
+      "epoch": 0.8843537414965986,
+      "grad_norm": 3.921875,
+      "learning_rate": 4.0756507127038494e-07,
+      "loss": 0.0075,
+      "mean_token_accuracy": 0.9979239463806152,
+      "num_tokens": 31953781.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.627412760257721,
+      "epoch": 0.8881330309901738,
+      "grad_norm": 1.171875,
+      "learning_rate": 3.8186619582565974e-07,
+      "loss": 0.0023,
+      "mean_token_accuracy": 0.998963725566864,
+      "num_tokens": 32091277.0,
+      "step": 1175
+    },
+    {
+      "entropy": 0.6293660879135132,
+      "epoch": 0.891912320483749,
+      "grad_norm": 0.06640625,
+      "learning_rate": 3.569719685681405e-07,
+      "loss": 0.0017,
+      "mean_token_accuracy": 0.9989610433578491,
+      "num_tokens": 32227162.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.6281276702880859,
+      "epoch": 0.8956916099773242,
+      "grad_norm": 0.98828125,
+      "learning_rate": 3.3288672698471804e-07,
+      "loss": 0.0029,
+      "mean_token_accuracy": 0.9989610433578491,
+      "num_tokens": 32364049.0,
+      "step": 1185
+    },
+    {
+      "entropy": 0.6279310822486878,
+      "epoch": 0.8994708994708994,
+      "grad_norm": 1.71875,
+      "learning_rate": 3.096146676073253e-07,
+      "loss": 0.0046,
+      "mean_token_accuracy": 0.9978629589080811,
+      "num_tokens": 32500177.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.6304468154907227,
+      "epoch": 0.9032501889644747,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.8715984528174757e-07,
+      "loss": 0.0034,
+      "mean_token_accuracy": 0.9979340195655823,
+      "num_tokens": 32637884.0,
+      "step": 1195
+    },
+    {
+      "entropy": 0.6291935801506042,
+      "epoch": 0.9070294784580499,
+      "grad_norm": 0.173828125,
+      "learning_rate": 2.6552617246111966e-07,
+      "loss": 0.0013,
+      "mean_token_accuracy": 0.9994791626930237,
+      "num_tokens": 32775860.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.628238570690155,
+      "epoch": 0.9108087679516251,
+      "grad_norm": 0.25390625,
+      "learning_rate": 2.447174185242324e-07,
+      "loss": 0.0054,
+      "mean_token_accuracy": 0.9984263181686401,
+      "num_tokens": 32915185.0,
+      "step": 1205
+    },
+    {
+      "entropy": 0.6287149310111999,
+      "epoch": 0.9145880574452003,
+      "grad_norm": 0.3671875,
+      "learning_rate": 2.247372091187705e-07,
+      "loss": 0.0023,
+      "mean_token_accuracy": 0.9994910955429077,
+      "num_tokens": 33051666.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.6283146739006042,
+      "epoch": 0.9183673469387755,
+      "grad_norm": 1.78125,
+      "learning_rate": 2.0558902552959058e-07,
+      "loss": 0.0021,
+      "mean_token_accuracy": 0.9994652390480041,
+      "num_tokens": 33187522.0,
+      "step": 1215
+    },
+    {
+      "entropy": 0.6290141940116882,
+      "epoch": 0.9221466364323507,
+      "grad_norm": 0.384765625,
+      "learning_rate": 1.8727620407215375e-07,
+      "loss": 0.0009,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 33325395.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.6285704016685486,
+      "epoch": 0.9259259259259259,
+      "grad_norm": 4.90625,
+      "learning_rate": 1.6980193551121848e-07,
+      "loss": 0.0042,
+      "mean_token_accuracy": 0.9984615325927735,
+      "num_tokens": 33461743.0,
+      "step": 1225
+    },
+    {
+      "entropy": 0.6292840957641601,
+      "epoch": 0.9297052154195011,
+      "grad_norm": 1.1171875,
+      "learning_rate": 1.5316926450488878e-07,
+      "loss": 0.0031,
+      "mean_token_accuracy": 0.9994845390319824,
+      "num_tokens": 33599818.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.6308977961540222,
+      "epoch": 0.9334845049130763,
+      "grad_norm": 1.5390625,
+      "learning_rate": 1.373810890741284e-07,
+      "loss": 0.0121,
+      "mean_token_accuracy": 0.9984320402145386,
+      "num_tokens": 33736511.0,
+      "step": 1235
+    },
+    {
+      "entropy": 0.6314628601074219,
+      "epoch": 0.9372637944066515,
+      "grad_norm": 0.05224609375,
+      "learning_rate": 1.22440160097817e-07,
+      "loss": 0.0004,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 33875284.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.6279171347618103,
+      "epoch": 0.9410430839002267,
+      "grad_norm": 0.12353515625,
+      "learning_rate": 1.0834908083344253e-07,
+      "loss": 0.0004,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 34010539.0,
+      "step": 1245
+    },
+    {
+      "entropy": 0.6286433458328247,
+      "epoch": 0.9448223733938019,
+      "grad_norm": 0.09033203125,
+      "learning_rate": 9.511030646352615e-08,
+      "loss": 0.0057,
+      "mean_token_accuracy": 0.9989795923233032,
+      "num_tokens": 34146309.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.6301220536231995,
+      "epoch": 0.9486016628873771,
+      "grad_norm": 4.09375,
+      "learning_rate": 8.27261436678306e-08,
+      "loss": 0.0084,
+      "mean_token_accuracy": 0.9979233145713806,
+      "num_tokens": 34280942.0,
+      "step": 1255
+    },
+    {
+      "entropy": 0.6293729186058045,
+      "epoch": 0.9523809523809523,
+      "grad_norm": 0.265625,
+      "learning_rate": 7.11987502214595e-08,
+      "loss": 0.0006,
+      "mean_token_accuracy": 1.0,
+      "num_tokens": 34417174.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.6268362164497375,
+      "epoch": 0.9561602418745276,
+      "grad_norm": 1.4296875,
+      "learning_rate": 6.053013461889023e-08,
+      "loss": 0.0013,
+      "mean_token_accuracy": 0.9994764447212219,
+      "num_tokens": 34553146.0,
+      "step": 1265
+    },
+    {
+      "entropy": 0.626698911190033,
+      "epoch": 0.9599395313681028,
+      "grad_norm": 0.0306396484375,
+      "learning_rate": 5.072215572402006e-08,
+      "loss": 0.0009,
+      "mean_token_accuracy": 0.9994832038879394,
+      "num_tokens": 34690822.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.6287196755409241,
+      "epoch": 0.963718820861678,
+      "grad_norm": 1.0859375,
+      "learning_rate": 4.177652244628627e-08,
+      "loss": 0.0045,
+      "mean_token_accuracy": 0.9984400272369385,
+      "num_tokens": 34828094.0,
+      "step": 1275
+    },
+    {
+      "entropy": 0.6283414721488952,
+      "epoch": 0.9674981103552532,
+      "grad_norm": 0.765625,
+      "learning_rate": 3.369479344290938e-08,
+      "loss": 0.0036,
+      "mean_token_accuracy": 0.9989580750465393,
+      "num_tokens": 34965771.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.6298310279846191,
+      "epoch": 0.9712773998488284,
+      "grad_norm": 0.0947265625,
+      "learning_rate": 2.6478376847318687e-08,
+      "loss": 0.004,
+      "mean_token_accuracy": 0.9989501357078552,
+      "num_tokens": 35102076.0,
+      "step": 1285
+    },
+    {
+      "entropy": 0.6282585024833679,
+      "epoch": 0.9750566893424036,
+      "grad_norm": 0.0296630859375,
+      "learning_rate": 2.012853002380466e-08,
+      "loss": 0.0061,
+      "mean_token_accuracy": 0.9994832038879394,
+      "num_tokens": 35237791.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.6272670507431031,
+      "epoch": 0.9788359788359788,
+      "grad_norm": 5.25,
+      "learning_rate": 1.4646359348439165e-08,
+      "loss": 0.0024,
+      "mean_token_accuracy": 0.9994832038879394,
+      "num_tokens": 35375040.0,
+      "step": 1295
+    },
+    {
+      "entropy": 0.629937744140625,
+      "epoch": 0.982615268329554,
+      "grad_norm": 3.078125,
+      "learning_rate": 1.0032820016302458e-08,
+      "loss": 0.0056,
+      "mean_token_accuracy": 0.997934103012085,
+      "num_tokens": 35512390.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.6293532013893127,
+      "epoch": 0.9863945578231292,
+      "grad_norm": 0.291015625,
+      "learning_rate": 6.288715875057416e-09,
+      "loss": 0.0021,
+      "mean_token_accuracy": 0.9984112977981567,
+      "num_tokens": 35651051.0,
+      "step": 1305
+    },
+    {
+      "entropy": 0.629024076461792,
+      "epoch": 0.9901738473167044,
+      "grad_norm": 0.083984375,
+      "learning_rate": 3.41469928488547e-09,
+      "loss": 0.0043,
+      "mean_token_accuracy": 0.9984126091003418,
+      "num_tokens": 35790379.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.6272889018058777,
+      "epoch": 0.9939531368102796,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.4112710048247436e-09,
+      "loss": 0.0053,
+      "mean_token_accuracy": 0.9989569664001465,
+      "num_tokens": 35926573.0,
+      "step": 1315
+    },
+    {
+      "entropy": 0.6294793486595154,
+      "epoch": 0.9977324263038548,
+      "grad_norm": 2.828125,
+      "learning_rate": 2.787801055181838e-10,
+      "loss": 0.0093,
+      "mean_token_accuracy": 0.9979259252548218,
+      "num_tokens": 36062587.0,
+      "step": 1320
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1323,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.85961760271106e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e92b7ff4c9e2438b0e73ac23303f60236c8cd784fdcf00d6d4757cb29865cb14
+size 6417