Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{%- for message in messages -%}{%- if loop.first and message['role'] != 'system' -%}{{ '<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n' }}{%- endif -%}{{ '<|im_start|>' + message['role'] + '\\n' }}{%- if message['content'] is string -%}{{- message['content'] -}}{%- elif message['content'] is iterable -%}{%- for item in message['content'] -%}{%- if item['type'].startswith('image') -%}{%- set image_count.value = image_count.value + 1 -%}{%- if add_vision_id -%}{{ 'Picture ' + image_count.value + ': ' }}{%- endif -%}{{ '<|vision_start|><|image_pad|><|vision_end|>' }}{%- elif item['type'].startswith('video') -%}{%- set video_count.value = video_count.value + 1 -%}{%- if add_vision_id -%}{{ 'Video ' + video_count.value + ': ' }}{%- endif -%}{{ '<|vision_start|><|video_pad|><|vision_end|>' }}{%- elif item['type']=='text' -%}{{- item['text'] if 'text' in item else item['content'] -}}{%- endif -%}{%- endfor -%}{%- endif -%}{{ '<|im_end|>\\n' }}{%- endfor -%}{%- if add_generation_prompt -%}{{- '<|im_start|>assistant\\n' -}}{%- endif -%}"
+}

config.json

@@ -0,0 +1,65 @@
+{
+  "architectures": [
+    "Qwen2_5_VLForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "image_token_id": 151655,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 128000,
+  "max_window_layers": 28,
+  "model_type": "qwen2_5_vl",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "mrope_section": [
+      16,
+      24,
+      24
+    ],
+    "rope_type": "default",
+    "type": "default"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "video_token_id": 151656,
+  "vision_config": {
+    "depth": 32,
+    "fullatt_block_indexes": [
+      7,
+      15,
+      23,
+      31
+    ],
+    "hidden_act": "silu",
+    "hidden_size": 1280,
+    "in_channels": 3,
+    "in_chans": 3,
+    "intermediate_size": 3420,
+    "model_type": "qwen2_5_vl",
+    "num_heads": 16,
+    "out_hidden_size": 3584,
+    "patch_size": 14,
+    "spatial_merge_size": 2,
+    "spatial_patch_size": 14,
+    "temporal_patch_size": 2,
+    "tokens_per_second": 2,
+    "torch_dtype": "bfloat16",
+    "window_size": 112
+  },
+  "vision_end_token_id": 151653,
+  "vision_start_token_id": 151652,
+  "vision_token_id": 151654,
+  "vocab_size": 152064
+}

generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 1e-06,
+  "transformers_version": "4.51.3"
+}

logs/rank_0001.log

@@ -0,0 +1,157 @@
+[2025-07-09 13:32:43,835][oumi][rank1][pid:1132651][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 13:32:45,405][oumi][rank1][pid:1132651][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 13:32:49,854][oumi][rank1][pid:1132651][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 13:32:49,854][oumi][rank1][pid:1132651][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 13:32:50,450][oumi][rank1][pid:1132651][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 13:32:52,746][oumi][rank1][pid:1132651][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 13:33:04,003][oumi][rank1][pid:1132651][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=1, local_world_size=8, local_rank=1)
+[2025-07-09 13:33:04,004][oumi][rank1][pid:1132651][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2DecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 13:33:04,005][oumi][rank1][pid:1132651][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 13:33:04,005][oumi][rank1][pid:1132651][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=1, local_world_size=8, local_rank=1))...
+[2025-07-09 13:33:04,048][oumi][rank1][pid:1132651][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 13:33:07,154][oumi][rank1][pid:1132651][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 13:33:07,154][oumi][rank1][pid:1132651][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 13:33:07,574][oumi][rank1][pid:1132651][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.549, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 13:33:07,780][oumi][rank1][pid:1132651][MainThread][INFO]][train.py:506] Training init time: 24.023s
+[2025-07-09 13:33:07,781][oumi][rank1][pid:1132651][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 13:53:48,662][oumi][rank1][pid:1137608][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 13:53:50,199][oumi][rank1][pid:1137608][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 13:53:54,736][oumi][rank1][pid:1137608][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 13:53:54,736][oumi][rank1][pid:1137608][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 13:53:55,283][oumi][rank1][pid:1137608][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 13:53:57,671][oumi][rank1][pid:1137608][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 13:54:09,973][oumi][rank1][pid:1137608][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=1, local_world_size=8, local_rank=1)
+[2025-07-09 13:54:09,974][oumi][rank1][pid:1137608][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2DecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 13:54:09,974][oumi][rank1][pid:1137608][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 13:54:09,974][oumi][rank1][pid:1137608][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=1, local_world_size=8, local_rank=1))...
+[2025-07-09 13:54:10,001][oumi][rank1][pid:1137608][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 13:54:13,231][oumi][rank1][pid:1137608][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 13:54:13,232][oumi][rank1][pid:1137608][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 13:54:13,750][oumi][rank1][pid:1137608][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.48100000000001, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 13:54:13,825][oumi][rank1][pid:1137608][MainThread][INFO]][train.py:506] Training init time: 25.227s
+[2025-07-09 13:54:13,825][oumi][rank1][pid:1137608][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 14:02:11,522][oumi][rank1][pid:1141190][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 14:02:13,069][oumi][rank1][pid:1141190][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 14:02:17,581][oumi][rank1][pid:1141190][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 14:02:17,581][oumi][rank1][pid:1141190][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 14:02:18,113][oumi][rank1][pid:1141190][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 14:02:20,380][oumi][rank1][pid:1141190][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 14:02:31,501][oumi][rank1][pid:1141190][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=1, local_world_size=8, local_rank=1)
+[2025-07-09 14:02:31,501][oumi][rank1][pid:1141190][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'HYBRID_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'ACCELERATE_MIXED_PRECISION': 'bf16', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'SIZE_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 14:02:31,501][oumi][rank1][pid:1141190][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 14:02:31,501][oumi][rank1][pid:1141190][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=1, local_world_size=8, local_rank=1))...
+[2025-07-09 14:02:31,537][oumi][rank1][pid:1141190][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 14:02:34,872][oumi][rank1][pid:1141190][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 14:02:34,872][oumi][rank1][pid:1141190][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 14:02:35,399][oumi][rank1][pid:1141190][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.53, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 14:02:35,478][oumi][rank1][pid:1141190][MainThread][INFO]][train.py:506] Training init time: 24.021s
+[2025-07-09 14:02:35,478][oumi][rank1][pid:1141190][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 14:55:03,136][oumi][rank1][pid:1152359][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 14:55:04,715][oumi][rank1][pid:1152359][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 14:55:09,245][oumi][rank1][pid:1152359][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 14:55:09,245][oumi][rank1][pid:1152359][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/limo-vis-mid-resize'
+[2025-07-09 14:55:10,669][oumi][rank1][pid:1152359][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 49430293
+	Download size: 43750929
+	Size: 93181222 bytes
+	Rows: 598
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 14:55:12,915][oumi][rank1][pid:1152359][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (598, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 14:55:24,721][oumi][rank1][pid:1152359][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=1, local_world_size=8, local_rank=1)
+[2025-07-09 14:55:24,721][oumi][rank1][pid:1152359][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'HYBRID_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'ACCELERATE_MIXED_PRECISION': 'bf16', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'SIZE_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 14:55:24,721][oumi][rank1][pid:1152359][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 14:55:24,721][oumi][rank1][pid:1152359][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=1, local_world_size=8, local_rank=1))...
+[2025-07-09 14:55:24,747][oumi][rank1][pid:1152359][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 14:55:27,956][oumi][rank1][pid:1152359][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 14:55:27,956][oumi][rank1][pid:1152359][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 14:55:28,392][oumi][rank1][pid:1152359][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6338.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.402, power_limit_watts=700.0, gpu_utilization=1, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 14:55:28,466][oumi][rank1][pid:1152359][MainThread][INFO]][train.py:506] Training init time: 25.399s
+[2025-07-09 14:55:28,466][oumi][rank1][pid:1152359][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 15:16:43,473][oumi][rank1][pid:1159829][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 15:16:44,967][oumi][rank1][pid:1159829][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 15:16:49,528][oumi][rank1][pid:1159829][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 15:16:49,528][oumi][rank1][pid:1159829][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/limo-vis-mid-resize'
+[2025-07-09 15:16:50,147][oumi][rank1][pid:1159829][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 49430293
+	Download size: 43750929
+	Size: 93181222 bytes
+	Rows: 598
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 15:16:52,251][oumi][rank1][pid:1159829][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (598, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 15:17:03,877][oumi][rank1][pid:1159829][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=1, local_world_size=8, local_rank=1)
+[2025-07-09 15:17:03,878][oumi][rank1][pid:1159829][MainThread][INFO]][distributed.py:481] 'FSDP_TRANSFORMER_CLS_TO_WRAP' is set to 'Qwen2_5_VLDecoderLayer' based on 'transformers.models.qwen2_5_vl.modeling_qwen2_5_vl.Qwen2_5_VLDecoderLayer'.
+[2025-07-09 15:17:03,878][oumi][rank1][pid:1159829][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2_5_VLDecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 15:17:03,878][oumi][rank1][pid:1159829][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 15:17:03,878][oumi][rank1][pid:1159829][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=1, local_world_size=8, local_rank=1))...
+[2025-07-09 15:17:03,906][oumi][rank1][pid:1159829][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 15:17:07,484][oumi][rank1][pid:1159829][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 15:17:07,484][oumi][rank1][pid:1159829][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 15:17:07,905][oumi][rank1][pid:1159829][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.33, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 15:17:07,982][oumi][rank1][pid:1159829][MainThread][INFO]][train.py:506] Training init time: 24.574s
+[2025-07-09 15:17:07,982][oumi][rank1][pid:1159829][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 17:00:38,964][oumi][rank1][pid:1159829][MainThread][INFO]][train.py:514] Training is Complete.
+[2025-07-09 17:00:38,970][oumi][rank1][pid:1159829][MainThread][INFO]][device_utils.py:297] GPU Metrics After Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=42828.0, temperature=32, fan_speed=None, fan_speeds=None, power_usage_watts=122.708, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 17:00:38,970][oumi][rank1][pid:1159829][MainThread][INFO]][torch_utils.py:136] Peak GPU memory usage: 36.26 GB
+[2025-07-09 17:00:38,970][oumi][rank1][pid:1159829][MainThread][INFO]][train.py:521] Saving final state...
+[2025-07-09 17:00:39,134][oumi][rank1][pid:1159829][MainThread][INFO]][train.py:526] Saving final model...
+[2025-07-09 17:00:39,134][oumi][rank1][pid:1159829][MainThread][INFO]][hf_trainer.py:142] Saving FULL_STATE_DICT for final model checkpoint.
+[2025-07-09 17:01:54,953][oumi][rank1][pid:1159829][MainThread][INFO]][hf_trainer.py:149] Model has been saved at output/qwen2_5_vl_7b_limo_vis
+[2025-07-09 17:01:58,289][oumi][rank1][pid:1159829][MainThread][INFO]][hf_trainer.py:153] Processor config has been saved at output/qwen2_5_vl_7b_limo_vis
+[2025-07-09 17:03:28,865][oumi][rank1][pid:1159829][MainThread][INFO]][train.py:214] 
+
+» We're always looking for feedback. What's one thing we can improve? https://oumi.ai/feedback

logs/rank_0002.log

@@ -0,0 +1,157 @@
+[2025-07-09 13:32:43,847][oumi][rank2][pid:1132652][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 13:32:45,425][oumi][rank2][pid:1132652][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 13:32:49,847][oumi][rank2][pid:1132652][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 13:32:49,847][oumi][rank2][pid:1132652][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 13:32:50,390][oumi][rank2][pid:1132652][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 13:32:52,711][oumi][rank2][pid:1132652][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 13:33:03,995][oumi][rank2][pid:1132652][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=2, local_world_size=8, local_rank=2)
+[2025-07-09 13:33:03,996][oumi][rank2][pid:1132652][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2DecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 13:33:03,996][oumi][rank2][pid:1132652][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 13:33:03,996][oumi][rank2][pid:1132652][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=2, local_world_size=8, local_rank=2))...
+[2025-07-09 13:33:04,023][oumi][rank2][pid:1132652][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 13:33:07,183][oumi][rank2][pid:1132652][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 13:33:07,184][oumi][rank2][pid:1132652][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 13:33:07,638][oumi][rank2][pid:1132652][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.549, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 13:33:07,782][oumi][rank2][pid:1132652][MainThread][INFO]][train.py:506] Training init time: 24.027s
+[2025-07-09 13:33:07,782][oumi][rank2][pid:1132652][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 13:53:48,675][oumi][rank2][pid:1137609][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 13:53:50,217][oumi][rank2][pid:1137609][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 13:53:54,736][oumi][rank2][pid:1137609][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 13:53:54,736][oumi][rank2][pid:1137609][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 13:53:55,346][oumi][rank2][pid:1137609][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 13:53:57,690][oumi][rank2][pid:1137609][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 13:54:09,980][oumi][rank2][pid:1137609][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=2, local_world_size=8, local_rank=2)
+[2025-07-09 13:54:09,980][oumi][rank2][pid:1137609][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2DecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 13:54:09,981][oumi][rank2][pid:1137609][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 13:54:09,981][oumi][rank2][pid:1137609][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=2, local_world_size=8, local_rank=2))...
+[2025-07-09 13:54:10,013][oumi][rank2][pid:1137609][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 13:54:13,246][oumi][rank2][pid:1137609][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 13:54:13,247][oumi][rank2][pid:1137609][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 13:54:13,737][oumi][rank2][pid:1137609][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.48100000000001, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 13:54:13,841][oumi][rank2][pid:1137609][MainThread][INFO]][train.py:506] Training init time: 25.243s
+[2025-07-09 13:54:13,841][oumi][rank2][pid:1137609][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 14:02:11,525][oumi][rank2][pid:1141191][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 14:02:13,044][oumi][rank2][pid:1141191][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 14:02:17,598][oumi][rank2][pid:1141191][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 14:02:17,598][oumi][rank2][pid:1141191][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 14:02:18,162][oumi][rank2][pid:1141191][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 14:02:20,509][oumi][rank2][pid:1141191][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 14:02:31,490][oumi][rank2][pid:1141191][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=2, local_world_size=8, local_rank=2)
+[2025-07-09 14:02:31,490][oumi][rank2][pid:1141191][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'HYBRID_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'ACCELERATE_MIXED_PRECISION': 'bf16', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'SIZE_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 14:02:31,490][oumi][rank2][pid:1141191][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 14:02:31,490][oumi][rank2][pid:1141191][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=2, local_world_size=8, local_rank=2))...
+[2025-07-09 14:02:31,519][oumi][rank2][pid:1141191][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 14:02:34,612][oumi][rank2][pid:1141191][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 14:02:34,612][oumi][rank2][pid:1141191][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 14:02:35,133][oumi][rank2][pid:1141191][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.561, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 14:02:35,493][oumi][rank2][pid:1141191][MainThread][INFO]][train.py:506] Training init time: 24.035s
+[2025-07-09 14:02:35,493][oumi][rank2][pid:1141191][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 14:55:03,146][oumi][rank2][pid:1152360][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 14:55:04,708][oumi][rank2][pid:1152360][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 14:55:09,286][oumi][rank2][pid:1152360][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 14:55:09,286][oumi][rank2][pid:1152360][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/limo-vis-mid-resize'
+[2025-07-09 14:55:10,627][oumi][rank2][pid:1152360][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 49430293
+	Download size: 43750929
+	Size: 93181222 bytes
+	Rows: 598
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 14:55:12,898][oumi][rank2][pid:1152360][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (598, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 14:55:24,724][oumi][rank2][pid:1152360][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=2, local_world_size=8, local_rank=2)
+[2025-07-09 14:55:24,725][oumi][rank2][pid:1152360][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'HYBRID_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'ACCELERATE_MIXED_PRECISION': 'bf16', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'SIZE_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 14:55:24,725][oumi][rank2][pid:1152360][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 14:55:24,725][oumi][rank2][pid:1152360][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=2, local_world_size=8, local_rank=2))...
+[2025-07-09 14:55:24,821][oumi][rank2][pid:1152360][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 14:55:27,954][oumi][rank2][pid:1152360][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 14:55:27,954][oumi][rank2][pid:1152360][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 14:55:28,414][oumi][rank2][pid:1152360][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6338.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.402, power_limit_watts=700.0, gpu_utilization=1, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 14:55:28,462][oumi][rank2][pid:1152360][MainThread][INFO]][train.py:506] Training init time: 25.395s
+[2025-07-09 14:55:28,477][oumi][rank2][pid:1152360][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 15:16:43,480][oumi][rank2][pid:1159830][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 15:16:45,066][oumi][rank2][pid:1159830][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 15:16:49,619][oumi][rank2][pid:1159830][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 15:16:49,619][oumi][rank2][pid:1159830][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/limo-vis-mid-resize'
+[2025-07-09 15:16:50,266][oumi][rank2][pid:1159830][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 49430293
+	Download size: 43750929
+	Size: 93181222 bytes
+	Rows: 598
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 15:16:52,368][oumi][rank2][pid:1159830][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (598, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 15:17:03,885][oumi][rank2][pid:1159830][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=2, local_world_size=8, local_rank=2)
+[2025-07-09 15:17:03,885][oumi][rank2][pid:1159830][MainThread][INFO]][distributed.py:481] 'FSDP_TRANSFORMER_CLS_TO_WRAP' is set to 'Qwen2_5_VLDecoderLayer' based on 'transformers.models.qwen2_5_vl.modeling_qwen2_5_vl.Qwen2_5_VLDecoderLayer'.
+[2025-07-09 15:17:03,885][oumi][rank2][pid:1159830][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2_5_VLDecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 15:17:03,885][oumi][rank2][pid:1159830][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 15:17:03,885][oumi][rank2][pid:1159830][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=2, local_world_size=8, local_rank=2))...
+[2025-07-09 15:17:03,924][oumi][rank2][pid:1159830][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 15:17:06,935][oumi][rank2][pid:1159830][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 15:17:06,935][oumi][rank2][pid:1159830][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 15:17:07,335][oumi][rank2][pid:1159830][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.363, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 15:17:07,977][oumi][rank2][pid:1159830][MainThread][INFO]][train.py:506] Training init time: 24.566s
+[2025-07-09 15:17:07,977][oumi][rank2][pid:1159830][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 17:00:39,124][oumi][rank2][pid:1159830][MainThread][INFO]][train.py:514] Training is Complete.
+[2025-07-09 17:00:39,131][oumi][rank2][pid:1159830][MainThread][INFO]][device_utils.py:297] GPU Metrics After Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=42828.0, temperature=32, fan_speed=None, fan_speeds=None, power_usage_watts=122.926, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 17:00:39,131][oumi][rank2][pid:1159830][MainThread][INFO]][torch_utils.py:136] Peak GPU memory usage: 36.25 GB
+[2025-07-09 17:00:39,131][oumi][rank2][pid:1159830][MainThread][INFO]][train.py:521] Saving final state...
+[2025-07-09 17:00:39,135][oumi][rank2][pid:1159830][MainThread][INFO]][train.py:526] Saving final model...
+[2025-07-09 17:00:39,135][oumi][rank2][pid:1159830][MainThread][INFO]][hf_trainer.py:142] Saving FULL_STATE_DICT for final model checkpoint.
+[2025-07-09 17:01:54,948][oumi][rank2][pid:1159830][MainThread][INFO]][hf_trainer.py:149] Model has been saved at output/qwen2_5_vl_7b_limo_vis
+[2025-07-09 17:01:58,584][oumi][rank2][pid:1159830][MainThread][INFO]][hf_trainer.py:153] Processor config has been saved at output/qwen2_5_vl_7b_limo_vis
+[2025-07-09 17:03:28,876][oumi][rank2][pid:1159830][MainThread][INFO]][train.py:214] 
+
+» We're always looking for feedback. What's one thing we can improve? https://oumi.ai/feedback

logs/rank_0003.log

@@ -0,0 +1,157 @@
+[2025-07-09 13:32:43,852][oumi][rank3][pid:1132653][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 13:32:45,401][oumi][rank3][pid:1132653][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 13:32:49,823][oumi][rank3][pid:1132653][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 13:32:49,823][oumi][rank3][pid:1132653][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 13:32:50,392][oumi][rank3][pid:1132653][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 13:32:52,757][oumi][rank3][pid:1132653][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 13:33:04,008][oumi][rank3][pid:1132653][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=3, local_world_size=8, local_rank=3)
+[2025-07-09 13:33:04,008][oumi][rank3][pid:1132653][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2DecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 13:33:04,008][oumi][rank3][pid:1132653][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 13:33:04,008][oumi][rank3][pid:1132653][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=3, local_world_size=8, local_rank=3))...
+[2025-07-09 13:33:04,066][oumi][rank3][pid:1132653][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 13:33:07,090][oumi][rank3][pid:1132653][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 13:33:07,091][oumi][rank3][pid:1132653][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 13:33:07,595][oumi][rank3][pid:1132653][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.549, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 13:33:07,781][oumi][rank3][pid:1132653][MainThread][INFO]][train.py:506] Training init time: 24.027s
+[2025-07-09 13:33:07,782][oumi][rank3][pid:1132653][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 13:53:48,672][oumi][rank3][pid:1137610][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 13:53:50,176][oumi][rank3][pid:1137610][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 13:53:54,738][oumi][rank3][pid:1137610][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 13:53:54,738][oumi][rank3][pid:1137610][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 13:53:55,321][oumi][rank3][pid:1137610][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 13:53:57,696][oumi][rank3][pid:1137610][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 13:54:09,968][oumi][rank3][pid:1137610][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=3, local_world_size=8, local_rank=3)
+[2025-07-09 13:54:09,968][oumi][rank3][pid:1137610][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2DecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 13:54:09,968][oumi][rank3][pid:1137610][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 13:54:09,968][oumi][rank3][pid:1137610][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=3, local_world_size=8, local_rank=3))...
+[2025-07-09 13:54:09,997][oumi][rank3][pid:1137610][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 13:54:13,202][oumi][rank3][pid:1137610][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 13:54:13,213][oumi][rank3][pid:1137610][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 13:54:13,742][oumi][rank3][pid:1137610][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.48100000000001, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 13:54:13,825][oumi][rank3][pid:1137610][MainThread][INFO]][train.py:506] Training init time: 25.224s
+[2025-07-09 13:54:13,825][oumi][rank3][pid:1137610][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 14:02:11,518][oumi][rank3][pid:1141192][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 14:02:13,076][oumi][rank3][pid:1141192][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 14:02:17,589][oumi][rank3][pid:1141192][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 14:02:17,589][oumi][rank3][pid:1141192][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 14:02:18,053][oumi][rank3][pid:1141192][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 14:02:20,422][oumi][rank3][pid:1141192][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 14:02:31,489][oumi][rank3][pid:1141192][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=3, local_world_size=8, local_rank=3)
+[2025-07-09 14:02:31,489][oumi][rank3][pid:1141192][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'HYBRID_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'ACCELERATE_MIXED_PRECISION': 'bf16', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'SIZE_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 14:02:31,489][oumi][rank3][pid:1141192][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 14:02:31,489][oumi][rank3][pid:1141192][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=3, local_world_size=8, local_rank=3))...
+[2025-07-09 14:02:31,517][oumi][rank3][pid:1141192][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 14:02:34,788][oumi][rank3][pid:1141192][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 14:02:34,788][oumi][rank3][pid:1141192][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 14:02:35,378][oumi][rank3][pid:1141192][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.536, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 14:02:35,482][oumi][rank3][pid:1141192][MainThread][INFO]][train.py:506] Training init time: 24.020s
+[2025-07-09 14:02:35,482][oumi][rank3][pid:1141192][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 14:55:03,122][oumi][rank3][pid:1152361][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 14:55:04,688][oumi][rank3][pid:1152361][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 14:55:09,245][oumi][rank3][pid:1152361][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 14:55:09,246][oumi][rank3][pid:1152361][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/limo-vis-mid-resize'
+[2025-07-09 14:55:10,627][oumi][rank3][pid:1152361][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 49430293
+	Download size: 43750929
+	Size: 93181222 bytes
+	Rows: 598
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 14:55:12,906][oumi][rank3][pid:1152361][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (598, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 14:55:24,729][oumi][rank3][pid:1152361][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=3, local_world_size=8, local_rank=3)
+[2025-07-09 14:55:24,730][oumi][rank3][pid:1152361][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'HYBRID_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'ACCELERATE_MIXED_PRECISION': 'bf16', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'SIZE_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 14:55:24,730][oumi][rank3][pid:1152361][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 14:55:24,730][oumi][rank3][pid:1152361][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=3, local_world_size=8, local_rank=3))...
+[2025-07-09 14:55:24,761][oumi][rank3][pid:1152361][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 14:55:27,735][oumi][rank3][pid:1152361][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 14:55:27,735][oumi][rank3][pid:1152361][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 14:55:28,196][oumi][rank3][pid:1152361][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.405, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 14:55:28,462][oumi][rank3][pid:1152361][MainThread][INFO]][train.py:506] Training init time: 25.396s
+[2025-07-09 14:55:28,463][oumi][rank3][pid:1152361][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 15:16:43,465][oumi][rank3][pid:1159831][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 15:16:45,058][oumi][rank3][pid:1159831][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 15:16:49,520][oumi][rank3][pid:1159831][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 15:16:49,520][oumi][rank3][pid:1159831][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/limo-vis-mid-resize'
+[2025-07-09 15:16:50,184][oumi][rank3][pid:1159831][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 49430293
+	Download size: 43750929
+	Size: 93181222 bytes
+	Rows: 598
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 15:16:52,340][oumi][rank3][pid:1159831][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (598, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 15:17:03,892][oumi][rank3][pid:1159831][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=3, local_world_size=8, local_rank=3)
+[2025-07-09 15:17:03,892][oumi][rank3][pid:1159831][MainThread][INFO]][distributed.py:481] 'FSDP_TRANSFORMER_CLS_TO_WRAP' is set to 'Qwen2_5_VLDecoderLayer' based on 'transformers.models.qwen2_5_vl.modeling_qwen2_5_vl.Qwen2_5_VLDecoderLayer'.
+[2025-07-09 15:17:03,892][oumi][rank3][pid:1159831][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2_5_VLDecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 15:17:03,892][oumi][rank3][pid:1159831][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 15:17:03,892][oumi][rank3][pid:1159831][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=3, local_world_size=8, local_rank=3))...
+[2025-07-09 15:17:03,922][oumi][rank3][pid:1159831][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 15:17:07,479][oumi][rank3][pid:1159831][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 15:17:07,479][oumi][rank3][pid:1159831][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 15:17:07,903][oumi][rank3][pid:1159831][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.33, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 15:17:07,992][oumi][rank3][pid:1159831][MainThread][INFO]][train.py:506] Training init time: 24.580s
+[2025-07-09 15:17:07,992][oumi][rank3][pid:1159831][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 17:00:39,057][oumi][rank3][pid:1159831][MainThread][INFO]][train.py:514] Training is Complete.
+[2025-07-09 17:00:39,062][oumi][rank3][pid:1159831][MainThread][INFO]][device_utils.py:297] GPU Metrics After Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=42828.0, temperature=32, fan_speed=None, fan_speeds=None, power_usage_watts=122.837, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 17:00:39,062][oumi][rank3][pid:1159831][MainThread][INFO]][torch_utils.py:136] Peak GPU memory usage: 36.26 GB
+[2025-07-09 17:00:39,062][oumi][rank3][pid:1159831][MainThread][INFO]][train.py:521] Saving final state...
+[2025-07-09 17:00:39,136][oumi][rank3][pid:1159831][MainThread][INFO]][train.py:526] Saving final model...
+[2025-07-09 17:00:39,136][oumi][rank3][pid:1159831][MainThread][INFO]][hf_trainer.py:142] Saving FULL_STATE_DICT for final model checkpoint.
+[2025-07-09 17:01:54,955][oumi][rank3][pid:1159831][MainThread][INFO]][hf_trainer.py:149] Model has been saved at output/qwen2_5_vl_7b_limo_vis
+[2025-07-09 17:01:58,449][oumi][rank3][pid:1159831][MainThread][INFO]][hf_trainer.py:153] Processor config has been saved at output/qwen2_5_vl_7b_limo_vis
+[2025-07-09 17:03:28,891][oumi][rank3][pid:1159831][MainThread][INFO]][train.py:214] 
+
+» We're always looking for feedback. What's one thing we can improve? https://oumi.ai/feedback

logs/rank_0004.log

@@ -0,0 +1,157 @@
+[2025-07-09 13:32:43,844][oumi][rank4][pid:1132654][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 13:32:45,427][oumi][rank4][pid:1132654][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 13:32:49,810][oumi][rank4][pid:1132654][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 13:32:49,812][oumi][rank4][pid:1132654][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 13:32:50,387][oumi][rank4][pid:1132654][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 13:32:52,770][oumi][rank4][pid:1132654][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 13:33:04,017][oumi][rank4][pid:1132654][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=4, local_world_size=8, local_rank=4)
+[2025-07-09 13:33:04,018][oumi][rank4][pid:1132654][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2DecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 13:33:04,018][oumi][rank4][pid:1132654][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 13:33:04,018][oumi][rank4][pid:1132654][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=4, local_world_size=8, local_rank=4))...
+[2025-07-09 13:33:04,051][oumi][rank4][pid:1132654][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 13:33:07,260][oumi][rank4][pid:1132654][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 13:33:07,260][oumi][rank4][pid:1132654][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 13:33:07,631][oumi][rank4][pid:1132654][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.549, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 13:33:07,785][oumi][rank4][pid:1132654][MainThread][INFO]][train.py:506] Training init time: 24.029s
+[2025-07-09 13:33:07,785][oumi][rank4][pid:1132654][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 13:53:48,645][oumi][rank4][pid:1137611][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 13:53:50,205][oumi][rank4][pid:1137611][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 13:53:54,742][oumi][rank4][pid:1137611][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 13:53:54,742][oumi][rank4][pid:1137611][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 13:53:55,373][oumi][rank4][pid:1137611][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 13:53:57,712][oumi][rank4][pid:1137611][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 13:54:09,971][oumi][rank4][pid:1137611][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=4, local_world_size=8, local_rank=4)
+[2025-07-09 13:54:09,972][oumi][rank4][pid:1137611][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2DecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 13:54:09,972][oumi][rank4][pid:1137611][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 13:54:09,972][oumi][rank4][pid:1137611][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=4, local_world_size=8, local_rank=4))...
+[2025-07-09 13:54:10,006][oumi][rank4][pid:1137611][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 13:54:13,147][oumi][rank4][pid:1137611][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 13:54:13,147][oumi][rank4][pid:1137611][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 13:54:13,728][oumi][rank4][pid:1137611][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.48100000000001, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 13:54:13,831][oumi][rank4][pid:1137611][MainThread][INFO]][train.py:506] Training init time: 25.232s
+[2025-07-09 13:54:13,831][oumi][rank4][pid:1137611][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 14:02:11,511][oumi][rank4][pid:1141193][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 14:02:13,080][oumi][rank4][pid:1141193][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 14:02:17,595][oumi][rank4][pid:1141193][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 14:02:17,595][oumi][rank4][pid:1141193][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 14:02:18,086][oumi][rank4][pid:1141193][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 14:02:20,431][oumi][rank4][pid:1141193][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 14:02:31,504][oumi][rank4][pid:1141193][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=4, local_world_size=8, local_rank=4)
+[2025-07-09 14:02:31,504][oumi][rank4][pid:1141193][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'HYBRID_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'ACCELERATE_MIXED_PRECISION': 'bf16', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'SIZE_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 14:02:31,504][oumi][rank4][pid:1141193][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 14:02:31,505][oumi][rank4][pid:1141193][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=4, local_world_size=8, local_rank=4))...
+[2025-07-09 14:02:31,540][oumi][rank4][pid:1141193][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 14:02:34,964][oumi][rank4][pid:1141193][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 14:02:34,964][oumi][rank4][pid:1141193][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 14:02:35,352][oumi][rank4][pid:1141193][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.536, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 14:02:35,481][oumi][rank4][pid:1141193][MainThread][INFO]][train.py:506] Training init time: 24.021s
+[2025-07-09 14:02:35,481][oumi][rank4][pid:1141193][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 14:55:03,139][oumi][rank4][pid:1152362][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 14:55:04,672][oumi][rank4][pid:1152362][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 14:55:09,284][oumi][rank4][pid:1152362][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 14:55:09,284][oumi][rank4][pid:1152362][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/limo-vis-mid-resize'
+[2025-07-09 14:55:10,688][oumi][rank4][pid:1152362][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 49430293
+	Download size: 43750929
+	Size: 93181222 bytes
+	Rows: 598
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 14:55:12,884][oumi][rank4][pid:1152362][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (598, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 14:55:24,728][oumi][rank4][pid:1152362][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=4, local_world_size=8, local_rank=4)
+[2025-07-09 14:55:24,729][oumi][rank4][pid:1152362][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'HYBRID_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'ACCELERATE_MIXED_PRECISION': 'bf16', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'SIZE_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 14:55:24,729][oumi][rank4][pid:1152362][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 14:55:24,729][oumi][rank4][pid:1152362][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=4, local_world_size=8, local_rank=4))...
+[2025-07-09 14:55:24,755][oumi][rank4][pid:1152362][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 14:55:27,899][oumi][rank4][pid:1152362][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 14:55:27,900][oumi][rank4][pid:1152362][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 14:55:28,333][oumi][rank4][pid:1152362][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.39, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 14:55:28,462][oumi][rank4][pid:1152362][MainThread][INFO]][train.py:506] Training init time: 25.394s
+[2025-07-09 14:55:28,466][oumi][rank4][pid:1152362][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 15:16:43,476][oumi][rank4][pid:1159832][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 15:16:45,016][oumi][rank4][pid:1159832][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 15:16:49,573][oumi][rank4][pid:1159832][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 15:16:49,573][oumi][rank4][pid:1159832][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/limo-vis-mid-resize'
+[2025-07-09 15:16:50,096][oumi][rank4][pid:1159832][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 49430293
+	Download size: 43750929
+	Size: 93181222 bytes
+	Rows: 598
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 15:16:52,277][oumi][rank4][pid:1159832][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (598, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 15:17:03,880][oumi][rank4][pid:1159832][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=4, local_world_size=8, local_rank=4)
+[2025-07-09 15:17:03,880][oumi][rank4][pid:1159832][MainThread][INFO]][distributed.py:481] 'FSDP_TRANSFORMER_CLS_TO_WRAP' is set to 'Qwen2_5_VLDecoderLayer' based on 'transformers.models.qwen2_5_vl.modeling_qwen2_5_vl.Qwen2_5_VLDecoderLayer'.
+[2025-07-09 15:17:03,880][oumi][rank4][pid:1159832][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2_5_VLDecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 15:17:03,880][oumi][rank4][pid:1159832][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 15:17:03,880][oumi][rank4][pid:1159832][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=4, local_world_size=8, local_rank=4))...
+[2025-07-09 15:17:03,912][oumi][rank4][pid:1159832][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 15:17:07,075][oumi][rank4][pid:1159832][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 15:17:07,076][oumi][rank4][pid:1159832][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 15:17:07,360][oumi][rank4][pid:1159832][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.35600000000001, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 15:17:07,978][oumi][rank4][pid:1159832][MainThread][INFO]][train.py:506] Training init time: 24.566s
+[2025-07-09 15:17:07,978][oumi][rank4][pid:1159832][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 17:00:38,856][oumi][rank4][pid:1159832][MainThread][INFO]][train.py:514] Training is Complete.
+[2025-07-09 17:00:38,872][oumi][rank4][pid:1159832][MainThread][INFO]][device_utils.py:297] GPU Metrics After Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=42828.0, temperature=32, fan_speed=None, fan_speeds=None, power_usage_watts=122.72200000000001, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 17:00:38,872][oumi][rank4][pid:1159832][MainThread][INFO]][torch_utils.py:136] Peak GPU memory usage: 36.26 GB
+[2025-07-09 17:00:38,872][oumi][rank4][pid:1159832][MainThread][INFO]][train.py:521] Saving final state...
+[2025-07-09 17:00:39,135][oumi][rank4][pid:1159832][MainThread][INFO]][train.py:526] Saving final model...
+[2025-07-09 17:00:39,135][oumi][rank4][pid:1159832][MainThread][INFO]][hf_trainer.py:142] Saving FULL_STATE_DICT for final model checkpoint.
+[2025-07-09 17:01:54,952][oumi][rank4][pid:1159832][MainThread][INFO]][hf_trainer.py:149] Model has been saved at output/qwen2_5_vl_7b_limo_vis
+[2025-07-09 17:01:58,472][oumi][rank4][pid:1159832][MainThread][INFO]][hf_trainer.py:153] Processor config has been saved at output/qwen2_5_vl_7b_limo_vis
+[2025-07-09 17:03:28,890][oumi][rank4][pid:1159832][MainThread][INFO]][train.py:214] 
+
+» We're always looking for feedback. What's one thing we can improve? https://oumi.ai/feedback

logs/rank_0005.log

@@ -0,0 +1,157 @@
+[2025-07-09 13:32:43,832][oumi][rank5][pid:1132655][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 13:32:45,244][oumi][rank5][pid:1132655][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 13:32:49,695][oumi][rank5][pid:1132655][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 13:32:49,698][oumi][rank5][pid:1132655][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 13:32:50,317][oumi][rank5][pid:1132655][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 13:32:52,709][oumi][rank5][pid:1132655][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 13:33:04,014][oumi][rank5][pid:1132655][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=5, local_world_size=8, local_rank=5)
+[2025-07-09 13:33:04,015][oumi][rank5][pid:1132655][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2DecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 13:33:04,015][oumi][rank5][pid:1132655][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 13:33:04,015][oumi][rank5][pid:1132655][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=5, local_world_size=8, local_rank=5))...
+[2025-07-09 13:33:04,042][oumi][rank5][pid:1132655][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 13:33:07,007][oumi][rank5][pid:1132655][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 13:33:07,008][oumi][rank5][pid:1132655][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 13:33:07,545][oumi][rank5][pid:1132655][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.55, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 13:33:07,790][oumi][rank5][pid:1132655][MainThread][INFO]][train.py:506] Training init time: 24.035s
+[2025-07-09 13:33:07,790][oumi][rank5][pid:1132655][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 13:53:48,639][oumi][rank5][pid:1137612][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 13:53:50,200][oumi][rank5][pid:1137612][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 13:53:54,740][oumi][rank5][pid:1137612][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 13:53:54,741][oumi][rank5][pid:1137612][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 13:53:55,252][oumi][rank5][pid:1137612][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 13:53:57,650][oumi][rank5][pid:1137612][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 13:54:09,969][oumi][rank5][pid:1137612][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=5, local_world_size=8, local_rank=5)
+[2025-07-09 13:54:09,969][oumi][rank5][pid:1137612][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2DecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 13:54:09,969][oumi][rank5][pid:1137612][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 13:54:09,970][oumi][rank5][pid:1137612][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=5, local_world_size=8, local_rank=5))...
+[2025-07-09 13:54:09,998][oumi][rank5][pid:1137612][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 13:54:13,233][oumi][rank5][pid:1137612][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 13:54:13,233][oumi][rank5][pid:1137612][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 13:54:13,736][oumi][rank5][pid:1137612][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.48100000000001, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 13:54:13,829][oumi][rank5][pid:1137612][MainThread][INFO]][train.py:506] Training init time: 25.232s
+[2025-07-09 13:54:13,829][oumi][rank5][pid:1137612][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 14:02:11,508][oumi][rank5][pid:1141194][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 14:02:13,014][oumi][rank5][pid:1141194][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 14:02:17,341][oumi][rank5][pid:1141194][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 14:02:17,341][oumi][rank5][pid:1141194][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 14:02:17,916][oumi][rank5][pid:1141194][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 14:02:20,263][oumi][rank5][pid:1141194][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 14:02:31,487][oumi][rank5][pid:1141194][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=5, local_world_size=8, local_rank=5)
+[2025-07-09 14:02:31,487][oumi][rank5][pid:1141194][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'HYBRID_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'ACCELERATE_MIXED_PRECISION': 'bf16', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'SIZE_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 14:02:31,487][oumi][rank5][pid:1141194][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 14:02:31,487][oumi][rank5][pid:1141194][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=5, local_world_size=8, local_rank=5))...
+[2025-07-09 14:02:31,517][oumi][rank5][pid:1141194][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 14:02:34,953][oumi][rank5][pid:1141194][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 14:02:34,954][oumi][rank5][pid:1141194][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 14:02:35,422][oumi][rank5][pid:1141194][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.53, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 14:02:35,480][oumi][rank5][pid:1141194][MainThread][INFO]][train.py:506] Training init time: 24.023s
+[2025-07-09 14:02:35,480][oumi][rank5][pid:1141194][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 14:55:03,142][oumi][rank5][pid:1152363][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 14:55:04,642][oumi][rank5][pid:1152363][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 14:55:09,246][oumi][rank5][pid:1152363][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 14:55:09,246][oumi][rank5][pid:1152363][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/limo-vis-mid-resize'
+[2025-07-09 14:55:10,629][oumi][rank5][pid:1152363][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 49430293
+	Download size: 43750929
+	Size: 93181222 bytes
+	Rows: 598
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 14:55:12,868][oumi][rank5][pid:1152363][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (598, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 14:55:24,719][oumi][rank5][pid:1152363][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=5, local_world_size=8, local_rank=5)
+[2025-07-09 14:55:24,719][oumi][rank5][pid:1152363][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'HYBRID_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'ACCELERATE_MIXED_PRECISION': 'bf16', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'SIZE_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 14:55:24,720][oumi][rank5][pid:1152363][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 14:55:24,720][oumi][rank5][pid:1152363][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=5, local_world_size=8, local_rank=5))...
+[2025-07-09 14:55:24,774][oumi][rank5][pid:1152363][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 14:55:27,952][oumi][rank5][pid:1152363][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 14:55:27,953][oumi][rank5][pid:1152363][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 14:55:28,382][oumi][rank5][pid:1152363][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6168.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.402, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 14:55:28,467][oumi][rank5][pid:1152363][MainThread][INFO]][train.py:506] Training init time: 25.400s
+[2025-07-09 14:55:28,467][oumi][rank5][pid:1152363][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 15:16:43,455][oumi][rank5][pid:1159833][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 15:16:45,014][oumi][rank5][pid:1159833][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 15:16:49,590][oumi][rank5][pid:1159833][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 15:16:49,590][oumi][rank5][pid:1159833][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/limo-vis-mid-resize'
+[2025-07-09 15:16:50,159][oumi][rank5][pid:1159833][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 49430293
+	Download size: 43750929
+	Size: 93181222 bytes
+	Rows: 598
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 15:16:52,255][oumi][rank5][pid:1159833][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (598, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 15:17:03,888][oumi][rank5][pid:1159833][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=5, local_world_size=8, local_rank=5)
+[2025-07-09 15:17:03,888][oumi][rank5][pid:1159833][MainThread][INFO]][distributed.py:481] 'FSDP_TRANSFORMER_CLS_TO_WRAP' is set to 'Qwen2_5_VLDecoderLayer' based on 'transformers.models.qwen2_5_vl.modeling_qwen2_5_vl.Qwen2_5_VLDecoderLayer'.
+[2025-07-09 15:17:03,888][oumi][rank5][pid:1159833][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2_5_VLDecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 15:17:03,888][oumi][rank5][pid:1159833][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 15:17:03,888][oumi][rank5][pid:1159833][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=5, local_world_size=8, local_rank=5))...
+[2025-07-09 15:17:03,921][oumi][rank5][pid:1159833][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 15:17:07,368][oumi][rank5][pid:1159833][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 15:17:07,373][oumi][rank5][pid:1159833][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 15:17:07,811][oumi][rank5][pid:1159833][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.325, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 15:17:07,988][oumi][rank5][pid:1159833][MainThread][INFO]][train.py:506] Training init time: 24.579s
+[2025-07-09 15:17:07,988][oumi][rank5][pid:1159833][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 17:00:39,093][oumi][rank5][pid:1159833][MainThread][INFO]][train.py:514] Training is Complete.
+[2025-07-09 17:00:39,103][oumi][rank5][pid:1159833][MainThread][INFO]][device_utils.py:297] GPU Metrics After Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=42828.0, temperature=32, fan_speed=None, fan_speeds=None, power_usage_watts=122.837, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 17:00:39,104][oumi][rank5][pid:1159833][MainThread][INFO]][torch_utils.py:136] Peak GPU memory usage: 36.26 GB
+[2025-07-09 17:00:39,104][oumi][rank5][pid:1159833][MainThread][INFO]][train.py:521] Saving final state...
+[2025-07-09 17:00:39,135][oumi][rank5][pid:1159833][MainThread][INFO]][train.py:526] Saving final model...
+[2025-07-09 17:00:39,135][oumi][rank5][pid:1159833][MainThread][INFO]][hf_trainer.py:142] Saving FULL_STATE_DICT for final model checkpoint.
+[2025-07-09 17:01:54,948][oumi][rank5][pid:1159833][MainThread][INFO]][hf_trainer.py:149] Model has been saved at output/qwen2_5_vl_7b_limo_vis
+[2025-07-09 17:01:58,383][oumi][rank5][pid:1159833][MainThread][INFO]][hf_trainer.py:153] Processor config has been saved at output/qwen2_5_vl_7b_limo_vis
+[2025-07-09 17:03:28,891][oumi][rank5][pid:1159833][MainThread][INFO]][train.py:214] 
+
+» We're always looking for feedback. What's one thing we can improve? https://oumi.ai/feedback

logs/rank_0006.log

@@ -0,0 +1,157 @@
+[2025-07-09 13:32:43,824][oumi][rank6][pid:1132656][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 13:32:45,404][oumi][rank6][pid:1132656][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 13:32:49,852][oumi][rank6][pid:1132656][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 13:32:49,853][oumi][rank6][pid:1132656][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 13:32:50,377][oumi][rank6][pid:1132656][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 13:32:52,758][oumi][rank6][pid:1132656][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 13:33:04,016][oumi][rank6][pid:1132656][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=6, local_world_size=8, local_rank=6)
+[2025-07-09 13:33:04,018][oumi][rank6][pid:1132656][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2DecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 13:33:04,018][oumi][rank6][pid:1132656][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 13:33:04,018][oumi][rank6][pid:1132656][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=6, local_world_size=8, local_rank=6))...
+[2025-07-09 13:33:04,048][oumi][rank6][pid:1132656][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 13:33:07,264][oumi][rank6][pid:1132656][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 13:33:07,265][oumi][rank6][pid:1132656][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 13:33:07,740][oumi][rank6][pid:1132656][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.555, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 13:33:07,781][oumi][rank6][pid:1132656][MainThread][INFO]][train.py:506] Training init time: 24.022s
+[2025-07-09 13:33:07,781][oumi][rank6][pid:1132656][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 13:53:48,665][oumi][rank6][pid:1137613][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 13:53:50,199][oumi][rank6][pid:1137613][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 13:53:54,738][oumi][rank6][pid:1137613][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 13:53:54,738][oumi][rank6][pid:1137613][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 13:53:55,300][oumi][rank6][pid:1137613][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 13:53:57,672][oumi][rank6][pid:1137613][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 13:54:09,976][oumi][rank6][pid:1137613][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=6, local_world_size=8, local_rank=6)
+[2025-07-09 13:54:09,976][oumi][rank6][pid:1137613][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2DecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 13:54:09,977][oumi][rank6][pid:1137613][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 13:54:09,977][oumi][rank6][pid:1137613][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=6, local_world_size=8, local_rank=6))...
+[2025-07-09 13:54:10,003][oumi][rank6][pid:1137613][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 13:54:13,253][oumi][rank6][pid:1137613][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 13:54:13,253][oumi][rank6][pid:1137613][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 13:54:13,738][oumi][rank6][pid:1137613][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.48100000000001, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 13:54:13,833][oumi][rank6][pid:1137613][MainThread][INFO]][train.py:506] Training init time: 25.237s
+[2025-07-09 13:54:13,833][oumi][rank6][pid:1137613][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 14:02:11,522][oumi][rank6][pid:1141195][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 14:02:13,069][oumi][rank6][pid:1141195][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 14:02:17,584][oumi][rank6][pid:1141195][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 14:02:17,584][oumi][rank6][pid:1141195][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 14:02:18,140][oumi][rank6][pid:1141195][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 14:02:20,471][oumi][rank6][pid:1141195][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 14:02:31,498][oumi][rank6][pid:1141195][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=6, local_world_size=8, local_rank=6)
+[2025-07-09 14:02:31,499][oumi][rank6][pid:1141195][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'HYBRID_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'ACCELERATE_MIXED_PRECISION': 'bf16', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'SIZE_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 14:02:31,499][oumi][rank6][pid:1141195][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 14:02:31,499][oumi][rank6][pid:1141195][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=6, local_world_size=8, local_rank=6))...
+[2025-07-09 14:02:31,524][oumi][rank6][pid:1141195][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 14:02:34,783][oumi][rank6][pid:1141195][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 14:02:34,790][oumi][rank6][pid:1141195][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 14:02:35,255][oumi][rank6][pid:1141195][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.562, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 14:02:35,476][oumi][rank6][pid:1141195][MainThread][INFO]][train.py:506] Training init time: 24.015s
+[2025-07-09 14:02:35,477][oumi][rank6][pid:1141195][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 14:55:03,129][oumi][rank6][pid:1152364][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 14:55:04,681][oumi][rank6][pid:1152364][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 14:55:09,288][oumi][rank6][pid:1152364][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 14:55:09,288][oumi][rank6][pid:1152364][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/limo-vis-mid-resize'
+[2025-07-09 14:55:10,653][oumi][rank6][pid:1152364][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 49430293
+	Download size: 43750929
+	Size: 93181222 bytes
+	Rows: 598
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 14:55:12,862][oumi][rank6][pid:1152364][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (598, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 14:55:24,707][oumi][rank6][pid:1152364][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=6, local_world_size=8, local_rank=6)
+[2025-07-09 14:55:24,708][oumi][rank6][pid:1152364][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'HYBRID_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'ACCELERATE_MIXED_PRECISION': 'bf16', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'SIZE_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 14:55:24,708][oumi][rank6][pid:1152364][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 14:55:24,708][oumi][rank6][pid:1152364][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=6, local_world_size=8, local_rank=6))...
+[2025-07-09 14:55:24,735][oumi][rank6][pid:1152364][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 14:55:27,959][oumi][rank6][pid:1152364][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 14:55:27,959][oumi][rank6][pid:1152364][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 14:55:28,327][oumi][rank6][pid:1152364][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.39, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 14:55:28,470][oumi][rank6][pid:1152364][MainThread][INFO]][train.py:506] Training init time: 25.404s
+[2025-07-09 14:55:28,470][oumi][rank6][pid:1152364][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 15:16:43,459][oumi][rank6][pid:1159834][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 15:16:44,932][oumi][rank6][pid:1159834][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 15:16:49,575][oumi][rank6][pid:1159834][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 15:16:49,575][oumi][rank6][pid:1159834][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/limo-vis-mid-resize'
+[2025-07-09 15:16:50,122][oumi][rank6][pid:1159834][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 49430293
+	Download size: 43750929
+	Size: 93181222 bytes
+	Rows: 598
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 15:16:52,286][oumi][rank6][pid:1159834][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (598, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 15:17:03,878][oumi][rank6][pid:1159834][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=6, local_world_size=8, local_rank=6)
+[2025-07-09 15:17:03,878][oumi][rank6][pid:1159834][MainThread][INFO]][distributed.py:481] 'FSDP_TRANSFORMER_CLS_TO_WRAP' is set to 'Qwen2_5_VLDecoderLayer' based on 'transformers.models.qwen2_5_vl.modeling_qwen2_5_vl.Qwen2_5_VLDecoderLayer'.
+[2025-07-09 15:17:03,878][oumi][rank6][pid:1159834][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2_5_VLDecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 15:17:03,878][oumi][rank6][pid:1159834][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 15:17:03,878][oumi][rank6][pid:1159834][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=6, local_world_size=8, local_rank=6))...
+[2025-07-09 15:17:03,907][oumi][rank6][pid:1159834][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 15:17:07,408][oumi][rank6][pid:1159834][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 15:17:07,417][oumi][rank6][pid:1159834][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 15:17:07,907][oumi][rank6][pid:1159834][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.33, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 15:17:07,978][oumi][rank6][pid:1159834][MainThread][INFO]][train.py:506] Training init time: 24.570s
+[2025-07-09 15:17:07,978][oumi][rank6][pid:1159834][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 17:00:39,131][oumi][rank6][pid:1159834][MainThread][INFO]][train.py:514] Training is Complete.
+[2025-07-09 17:00:39,133][oumi][rank6][pid:1159834][MainThread][INFO]][device_utils.py:297] GPU Metrics After Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=42828.0, temperature=32, fan_speed=None, fan_speeds=None, power_usage_watts=122.926, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 17:00:39,133][oumi][rank6][pid:1159834][MainThread][INFO]][torch_utils.py:136] Peak GPU memory usage: 36.26 GB
+[2025-07-09 17:00:39,133][oumi][rank6][pid:1159834][MainThread][INFO]][train.py:521] Saving final state...
+[2025-07-09 17:00:39,134][oumi][rank6][pid:1159834][MainThread][INFO]][train.py:526] Saving final model...
+[2025-07-09 17:00:39,134][oumi][rank6][pid:1159834][MainThread][INFO]][hf_trainer.py:142] Saving FULL_STATE_DICT for final model checkpoint.
+[2025-07-09 17:01:54,948][oumi][rank6][pid:1159834][MainThread][INFO]][hf_trainer.py:149] Model has been saved at output/qwen2_5_vl_7b_limo_vis
+[2025-07-09 17:01:58,558][oumi][rank6][pid:1159834][MainThread][INFO]][hf_trainer.py:153] Processor config has been saved at output/qwen2_5_vl_7b_limo_vis
+[2025-07-09 17:03:28,891][oumi][rank6][pid:1159834][MainThread][INFO]][train.py:214] 
+
+» We're always looking for feedback. What's one thing we can improve? https://oumi.ai/feedback

logs/rank_0007.log

@@ -0,0 +1,157 @@
+[2025-07-09 13:32:43,828][oumi][rank7][pid:1132657][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 13:32:45,271][oumi][rank7][pid:1132657][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 13:32:49,688][oumi][rank7][pid:1132657][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 13:32:49,688][oumi][rank7][pid:1132657][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 13:32:50,318][oumi][rank7][pid:1132657][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 13:32:52,640][oumi][rank7][pid:1132657][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 13:33:03,997][oumi][rank7][pid:1132657][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=7, local_world_size=8, local_rank=7)
+[2025-07-09 13:33:03,998][oumi][rank7][pid:1132657][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2DecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 13:33:03,998][oumi][rank7][pid:1132657][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 13:33:03,998][oumi][rank7][pid:1132657][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=7, local_world_size=8, local_rank=7))...
+[2025-07-09 13:33:04,028][oumi][rank7][pid:1132657][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 13:33:07,155][oumi][rank7][pid:1132657][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 13:33:07,156][oumi][rank7][pid:1132657][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 13:33:07,624][oumi][rank7][pid:1132657][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.549, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 13:33:07,789][oumi][rank7][pid:1132657][MainThread][INFO]][train.py:506] Training init time: 24.034s
+[2025-07-09 13:33:07,789][oumi][rank7][pid:1132657][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 13:53:48,658][oumi][rank7][pid:1137614][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 13:53:50,195][oumi][rank7][pid:1137614][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 13:53:54,740][oumi][rank7][pid:1137614][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 13:53:54,740][oumi][rank7][pid:1137614][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 13:53:55,253][oumi][rank7][pid:1137614][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 13:53:57,621][oumi][rank7][pid:1137614][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 13:54:09,979][oumi][rank7][pid:1137614][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=7, local_world_size=8, local_rank=7)
+[2025-07-09 13:54:09,979][oumi][rank7][pid:1137614][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2DecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 13:54:09,979][oumi][rank7][pid:1137614][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 13:54:09,979][oumi][rank7][pid:1137614][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=7, local_world_size=8, local_rank=7))...
+[2025-07-09 13:54:10,007][oumi][rank7][pid:1137614][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 13:54:13,249][oumi][rank7][pid:1137614][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 13:54:13,250][oumi][rank7][pid:1137614][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 13:54:13,773][oumi][rank7][pid:1137614][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.486, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 13:54:13,825][oumi][rank7][pid:1137614][MainThread][INFO]][train.py:506] Training init time: 25.226s
+[2025-07-09 13:54:13,825][oumi][rank7][pid:1137614][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 14:02:11,503][oumi][rank7][pid:1141196][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 14:02:13,029][oumi][rank7][pid:1141196][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 14:02:17,485][oumi][rank7][pid:1141196][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 14:02:17,485][oumi][rank7][pid:1141196][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/GAIR_LIMO_img_v1'
+[2025-07-09 14:02:17,929][oumi][rank7][pid:1141196][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 86589729
+	Download size: 77602134
+	Size: 164191863 bytes
+	Rows: 778
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 14:02:20,292][oumi][rank7][pid:1141196][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (778, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 14:02:31,485][oumi][rank7][pid:1141196][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=7, local_world_size=8, local_rank=7)
+[2025-07-09 14:02:31,485][oumi][rank7][pid:1141196][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'HYBRID_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'ACCELERATE_MIXED_PRECISION': 'bf16', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'SIZE_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 14:02:31,485][oumi][rank7][pid:1141196][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 14:02:31,486][oumi][rank7][pid:1141196][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=7, local_world_size=8, local_rank=7))...
+[2025-07-09 14:02:31,514][oumi][rank7][pid:1141196][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 14:02:34,931][oumi][rank7][pid:1141196][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 14:02:34,931][oumi][rank7][pid:1141196][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 14:02:35,430][oumi][rank7][pid:1141196][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.53, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 14:02:35,487][oumi][rank7][pid:1141196][MainThread][INFO]][train.py:506] Training init time: 24.028s
+[2025-07-09 14:02:35,487][oumi][rank7][pid:1141196][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 14:55:03,125][oumi][rank7][pid:1152365][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 14:55:04,721][oumi][rank7][pid:1152365][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 14:55:09,250][oumi][rank7][pid:1152365][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 14:55:09,250][oumi][rank7][pid:1152365][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/limo-vis-mid-resize'
+[2025-07-09 14:55:10,699][oumi][rank7][pid:1152365][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 49430293
+	Download size: 43750929
+	Size: 93181222 bytes
+	Rows: 598
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 14:55:12,898][oumi][rank7][pid:1152365][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (598, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 14:55:24,715][oumi][rank7][pid:1152365][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=7, local_world_size=8, local_rank=7)
+[2025-07-09 14:55:24,715][oumi][rank7][pid:1152365][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'HYBRID_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'ACCELERATE_MIXED_PRECISION': 'bf16', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'SIZE_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 14:55:24,716][oumi][rank7][pid:1152365][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 14:55:24,716][oumi][rank7][pid:1152365][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=7, local_world_size=8, local_rank=7))...
+[2025-07-09 14:55:24,819][oumi][rank7][pid:1152365][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 14:55:28,005][oumi][rank7][pid:1152365][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 14:55:28,006][oumi][rank7][pid:1152365][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 14:55:28,381][oumi][rank7][pid:1152365][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6338.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.402, power_limit_watts=700.0, gpu_utilization=1, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 14:55:28,462][oumi][rank7][pid:1152365][MainThread][INFO]][train.py:506] Training init time: 25.395s
+[2025-07-09 14:55:28,462][oumi][rank7][pid:1152365][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 15:16:43,460][oumi][rank7][pid:1159835][MainThread][INFO]][models.py:433] Setting tokenizer to use the 'right' padding side for model 'Qwen/Qwen2.5-VL-7B-Instruct'. The 'right' padding side is configured as the default value for this model type.
+[2025-07-09 15:16:45,053][oumi][rank7][pid:1159835][MainThread][INFO]][models.py:479] Using the chat template 'qwen2-vl-instruct' specified in model config for model 'Qwen/Qwen2.5-VL-7B-Instruct'. 
+[2025-07-09 15:16:49,587][oumi][rank7][pid:1159835][MainThread][WARNING]][data.py:66] Using torchdata preprocessing pipeline. This is currently in beta and may not be stable.
+[2025-07-09 15:16:49,587][oumi][rank7][pid:1159835][MainThread][INFO]][base_map_dataset.py:91] Creating map dataset (type: HuggingFaceVisionDataset)... dataset_name: 'penfever/limo-vis-mid-resize'
+[2025-07-09 15:16:50,101][oumi][rank7][pid:1159835][MainThread][INFO]][base_map_dataset.py:487] Dataset Info:
+	Split: train
+	Version: 0.0.0
+	Dataset size: 49430293
+	Download size: 43750929
+	Size: 93181222 bytes
+	Rows: 598
+	Columns: ['question', 'answer', 'image_synthesis_code', 'solution', 'image']
+[2025-07-09 15:16:52,280][oumi][rank7][pid:1159835][MainThread][INFO]][base_map_dataset.py:426] Loaded DataFrame with shape: (598, 5). Columns:
+question                object
+answer                  object
+image_synthesis_code    object
+solution                object
+image                   object
+dtype: object
+[2025-07-09 15:17:03,895][oumi][rank7][pid:1159835][MainThread][INFO]][distributed.py:303] Initialized distributed (True): DeviceRankInfo(world_size=8, rank=7, local_world_size=8, local_rank=7)
+[2025-07-09 15:17:03,895][oumi][rank7][pid:1159835][MainThread][INFO]][distributed.py:481] 'FSDP_TRANSFORMER_CLS_TO_WRAP' is set to 'Qwen2_5_VLDecoderLayer' based on 'transformers.models.qwen2_5_vl.modeling_qwen2_5_vl.Qwen2_5_VLDecoderLayer'.
+[2025-07-09 15:17:03,895][oumi][rank7][pid:1159835][MainThread][INFO]][train.py:402] Set Accelerate environment variables for FSDP: {'ACCELERATE_DYNAMO_BACKEND': 'NO', 'ACCELERATE_DYNAMO_MODE': 'default', 'ACCELERATE_DYNAMO_USE_FULLGRAPH': 'False', 'ACCELERATE_DYNAMO_USE_DYNAMIC': 'False', 'FSDP_CPU_RAM_EFFICIENT_LOADING': 'true', 'FSDP_USE_ORIG_PARAMS': 'true', 'ACCELERATE_USE_FSDP': 'true', 'FSDP_SHARDING_STRATEGY': 'FULL_SHARD', 'FSDP_OFFLOAD_PARAMS': 'false', 'FSDP_BACKWARD_PREFETCH': 'BACKWARD_PRE', 'FSDP_FORWARD_PREFETCH': 'true', 'FSDP_STATE_DICT_TYPE': 'FULL_STATE_DICT', 'FSDP_AUTO_WRAP_POLICY': 'TRANSFORMER_BASED_WRAP', 'FSDP_MIN_NUM_PARAMS': '100000', 'FSDP_TRANSFORMER_CLS_TO_WRAP': 'Qwen2_5_VLDecoderLayer', 'FSDP_SYNC_MODULE_STATES': 'true', 'FSDP_ACTIVATION_CHECKPOINTING': 'true'}
+[2025-07-09 15:17:03,895][oumi][rank7][pid:1159835][MainThread][INFO]][models.py:219] Accelerate FSDP run detected! Setting device_map to None.
+[2025-07-09 15:17:03,895][oumi][rank7][pid:1159835][MainThread][INFO]][models.py:228] Building model using device_map: None (DeviceRankInfo(world_size=8, rank=7, local_world_size=8, local_rank=7))...
+[2025-07-09 15:17:03,924][oumi][rank7][pid:1159835][MainThread][INFO]][models.py:300] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForVision2Seq'> to instantiate model.
+[2025-07-09 15:17:07,428][oumi][rank7][pid:1159835][MainThread][INFO]][torch_profiler_utils.py:164] PROF: Torch Profiler disabled!
+[2025-07-09 15:17:07,428][oumi][rank7][pid:1159835][MainThread][WARNING]][callbacks.py:72] MFU logging requires packed datasets. Skipping MFU callbacks.
+[2025-07-09 15:17:07,856][oumi][rank7][pid:1159835][MainThread][INFO]][device_utils.py:297] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=6166.0, temperature=31, fan_speed=None, fan_speeds=None, power_usage_watts=122.33, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 15:17:07,982][oumi][rank7][pid:1159835][MainThread][INFO]][train.py:506] Training init time: 24.575s
+[2025-07-09 15:17:07,982][oumi][rank7][pid:1159835][MainThread][INFO]][train.py:507] Starting training... (TrainerType.TRL_SFT, transformers: 4.51.3)
+[2025-07-09 17:00:39,022][oumi][rank7][pid:1159835][MainThread][INFO]][train.py:514] Training is Complete.
+[2025-07-09 17:00:39,027][oumi][rank7][pid:1159835][MainThread][INFO]][device_utils.py:297] GPU Metrics After Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=8, used_memory_mb=42828.0, temperature=32, fan_speed=None, fan_speeds=None, power_usage_watts=122.837, power_limit_watts=700.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1980, clock_speed_sm=1980, clock_speed_memory=2619).
+[2025-07-09 17:00:39,027][oumi][rank7][pid:1159835][MainThread][INFO]][torch_utils.py:136] Peak GPU memory usage: 36.26 GB
+[2025-07-09 17:00:39,027][oumi][rank7][pid:1159835][MainThread][INFO]][train.py:521] Saving final state...
+[2025-07-09 17:00:39,135][oumi][rank7][pid:1159835][MainThread][INFO]][train.py:526] Saving final model...
+[2025-07-09 17:00:39,135][oumi][rank7][pid:1159835][MainThread][INFO]][hf_trainer.py:142] Saving FULL_STATE_DICT for final model checkpoint.
+[2025-07-09 17:01:54,957][oumi][rank7][pid:1159835][MainThread][INFO]][hf_trainer.py:149] Model has been saved at output/qwen2_5_vl_7b_limo_vis
+[2025-07-09 17:01:58,650][oumi][rank7][pid:1159835][MainThread][INFO]][hf_trainer.py:153] Processor config has been saved at output/qwen2_5_vl_7b_limo_vis
+[2025-07-09 17:03:28,908][oumi][rank7][pid:1159835][MainThread][INFO]][train.py:214] 
+
+» We're always looking for feedback. What's one thing we can improve? https://oumi.ai/feedback

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4b39025cc73539dc1591007c12c021a04f193f9d23244b52299a931e19fc0a3
+size 4968243304

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c16e51d30a854756e60d670eae891deb3296a579b3346ad609b2eb0b6b684ac
+size 4991495816

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6d8cef2edd97457b74a4f6446cdb30db6bf6bb73aa2c49c6b3df3cca1c48085
+size 4932751040

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:743f131ebdf4ab2f5181fd192a78ea993490fdfb466ba1ace58242e01151abf6
+size 1691924384

model.safetensors.index.json

@@ -0,0 +1,736 @@
+{
+  "metadata": {
+    "total_size": 16584333312
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors",
+    "visual.blocks.0.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.merger.ln_q.weight": "model-00001-of-00004.safetensors",
+    "visual.merger.mlp.0.bias": "model-00001-of-00004.safetensors",
+    "visual.merger.mlp.0.weight": "model-00001-of-00004.safetensors",
+    "visual.merger.mlp.2.bias": "model-00001-of-00004.safetensors",
+    "visual.merger.mlp.2.weight": "model-00001-of-00004.safetensors",
+    "visual.patch_embed.proj.weight": "model-00001-of-00004.safetensors"
+  }
+}

preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 12845056,
+  "merge_size": 2,
+  "min_pixels": 3136,
+  "patch_size": 14,
+  "processor_class": "Qwen2_5_VLProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 12845056,
+    "shortest_edge": 3136
+  },
+  "temporal_patch_size": 2
+}

runs/Jul09_14-02-35_oumi-compute002/events.out.tfevents.1752069998.oumi-compute002.1141189.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c2fe8292b359f7a7dac102e6cb0afd40194325f04c321c0a7a1da77d96ef445
+size 6313

runs/Jul09_14-55-28_oumi-compute002/events.out.tfevents.1752073113.oumi-compute002.1152358.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:889f9cfa6732da7bc9588e265577310b05b19dd155122a738ea3d6c7d6e12466
+size 6314

runs/Jul09_15-17-07_oumi-compute002/events.out.tfevents.1752074401.oumi-compute002.1159828.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbae9df3fc32959ab14cc92e5b8109c0e6afcbfa2951fd73d8e19d0b7c950622
+size 9679

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

telemetry/devices_info.txt

@@ -0,0 +1,9 @@
+CPU cores: 208 CUDA devices: 8
+device(0)='NVIDIA H100 80GB HBM3' Capability: (9, 0) Memory: [Total: 79.19GiB Free: 75.08GiB Allocated: 0.0GiB Cached: 0.0GiB]
+device(1)='NVIDIA H100 80GB HBM3' Capability: (9, 0) Memory: [Total: 79.19GiB Free: 78.67GiB Allocated: 0.0GiB Cached: 0.0GiB]
+device(2)='NVIDIA H100 80GB HBM3' Capability: (9, 0) Memory: [Total: 79.19GiB Free: 78.67GiB Allocated: 0.0GiB Cached: 0.0GiB]
+device(3)='NVIDIA H100 80GB HBM3' Capability: (9, 0) Memory: [Total: 79.19GiB Free: 78.67GiB Allocated: 0.0GiB Cached: 0.0GiB]
+device(4)='NVIDIA H100 80GB HBM3' Capability: (9, 0) Memory: [Total: 79.19GiB Free: 78.67GiB Allocated: 0.0GiB Cached: 0.0GiB]
+device(5)='NVIDIA H100 80GB HBM3' Capability: (9, 0) Memory: [Total: 79.19GiB Free: 78.67GiB Allocated: 0.0GiB Cached: 0.0GiB]
+device(6)='NVIDIA H100 80GB HBM3' Capability: (9, 0) Memory: [Total: 79.19GiB Free: 78.67GiB Allocated: 0.0GiB Cached: 0.0GiB]
+device(7)='NVIDIA H100 80GB HBM3' Capability: (9, 0) Memory: [Total: 79.19GiB Free: 78.67GiB Allocated: 0.0GiB Cached: 0.0GiB]

telemetry/telemetry_callback_metrics_rank0000.json

@@ -0,0 +1,12 @@
+{
+  "train_runtime": 6016.1372,
+  "train_samples_per_second": 0.499,
+  "train_steps_per_second": 0.062,
+  "train_tokens_per_second": 176.962,
+  "total_flos": 1.0562823256014848e+17,
+  "train_loss": 0.14466818872839213,
+  "num_tokens": 18179240.0,
+  "mean_token_accuracy": 1.0,
+  "epoch": 5.0,
+  "num_input_tokens_seen": 18179240
+}

telemetry/telemetry_callback_rank0000.json

@@ -0,0 +1,39 @@
+{
+  "hostname": "oumi-compute002",
+  "total_time": 6190.429405291099,
+  "timers": {
+    "epochs": {
+      "count": 5.0,
+      "mean": 1202.3129667625763,
+      "median": 1212.103585831821,
+      "std_dev": 23.461924971061983,
+      "min": 1162.4669050513767,
+      "max": 1221.4198807748035,
+      "total": 6011.564833812881,
+      "percentage": 97.11062739322513
+    },
+    "microsteps": {
+      "count": 373.0,
+      "mean": 12.383987677070104,
+      "median": 12.148281179368496,
+      "std_dev": 2.9678382288473095,
+      "min": 5.08648740593344,
+      "max": 24.33762474078685,
+      "total": 4619.227403547149,
+      "percentage": 74.61885276648162
+    },
+    "steps": {
+      "count": 373.0,
+      "mean": 12.383898228439365,
+      "median": 12.148281924892217,
+      "std_dev": 2.9679568977165727,
+      "min": 5.086486792191863,
+      "max": 24.337625022046268,
+      "total": 4619.194039207883,
+      "percentage": 74.6183138000048
+    }
+  },
+  "cuda_timers": {},
+  "gpu_memory": [],
+  "gpu_temperature": {}
+}

telemetry/telemetry_callback_wandb_rank0000.json

@@ -0,0 +1,5 @@
+{
+  "id": "fniivj34",
+  "name": "output/qwen2_5_vl_7b_limo_vis",
+  "url": "https://wandb.ai/nyu-dice-lab/huggingface/runs/fniivj34"
+}

telemetry/training_config.yaml

@@ -0,0 +1,206 @@
+data:
+  train:
+    datasets:
+    - dataset_name: hf_vision
+      dataset_path: null
+      subset: null
+      split: train
+      dataset_kwargs:
+        hf_dataset_path: penfever/limo-vis-mid-resize
+        image_column: image
+        question_column: question
+        answer_column: solution
+        return_tensors: true
+        processor_name: Qwen/Qwen2.5-VL-7B-Instruct
+        return_conversations: true
+      sample_count: null
+      mixture_proportion: null
+      shuffle: true
+      seed: 42
+      shuffle_buffer_size: 1000
+      trust_remote_code: true
+      transform_num_workers: auto
+    collator_name: vision_language_sft
+    collator_kwargs:
+      process_individually: true
+    pack: false
+    stream: false
+    target_col: null
+    mixture_strategy: first_exhausted
+    seed: null
+    use_async_dataset: false
+    use_torchdata: true
+  test:
+    datasets: []
+    collator_name: null
+    collator_kwargs: {}
+    pack: false
+    stream: false
+    target_col: null
+    mixture_strategy: first_exhausted
+    seed: null
+    use_async_dataset: false
+    use_torchdata: null
+  validation:
+    datasets: []
+    collator_name: null
+    collator_kwargs: {}
+    pack: false
+    stream: false
+    target_col: null
+    mixture_strategy: first_exhausted
+    seed: null
+    use_async_dataset: false
+    use_torchdata: null
+model:
+  model_name: Qwen/Qwen2.5-VL-7B-Instruct
+  adapter_model: null
+  tokenizer_name: null
+  tokenizer_pad_token: null
+  tokenizer_kwargs: {}
+  processor_kwargs: {}
+  model_max_length: 16384
+  load_pretrained_weights: true
+  trust_remote_code: true
+  torch_dtype_str: bfloat16
+  compile: false
+  chat_template: qwen2-vl-instruct
+  attn_implementation: flash_attention_2
+  device_map: auto
+  model_kwargs: {}
+  enable_liger_kernel: false
+  shard_for_eval: false
+  freeze_layers: []
+  model_revision: null
+training:
+  use_peft: false
+  trainer_type: TRL_SFT
+  enable_gradient_checkpointing: true
+  gradient_checkpointing_kwargs:
+    use_reentrant: false
+  output_dir: output/qwen2_5_vl_7b_limo_vis
+  per_device_train_batch_size: 1
+  per_device_eval_batch_size: 8
+  gradient_accumulation_steps: 1
+  max_steps: 375
+  num_train_epochs: 5
+  save_epoch: false
+  save_steps: 0
+  save_final_model: true
+  seed: 42
+  data_seed: 42
+  use_deterministic: false
+  full_determinism: false
+  run_name: null
+  metrics_function: null
+  reward_functions: null
+  grpo:
+    model_init_kwargs: {}
+    max_prompt_length: null
+    max_completion_length: null
+    num_generations: null
+    temperature: 0.9
+    remove_unused_columns: false
+    repetition_penalty: 1.0
+    use_vllm: false
+    vllm_mode: null
+    vllm_gpu_memory_utilization: 0.9
+    vllm_dtype: null
+    vllm_max_model_len: null
+    epsilon: 0.2
+    log_completions: false
+  log_level: info
+  dep_log_level: warning
+  enable_wandb: true
+  enable_mlflow: false
+  enable_tensorboard: true
+  logging_strategy: steps
+  logging_dir: null
+  logging_steps: 50
+  logging_first_step: false
+  eval_strategy: 'no'
+  eval_steps: 500
+  learning_rate: 2.0e-05
+  lr_scheduler_type: cosine
+  lr_scheduler_kwargs: {}
+  warmup_ratio: 0.03
+  warmup_steps: null
+  optimizer: adamw_torch_fused
+  weight_decay: 0.01
+  adam_beta1: 0.9
+  adam_beta2: 0.999
+  adam_epsilon: 1.0e-08
+  sgd_momentum: 0.0
+  mixed_precision_dtype: NONE
+  compile: false
+  include_performance_metrics: true
+  include_alternative_mfu_metrics: false
+  log_model_summary: false
+  resume_from_checkpoint: null
+  try_resume_from_last_checkpoint: false
+  dataloader_num_workers: 64
+  dataloader_persistent_workers: false
+  dataloader_prefetch_factor: 32
+  dataloader_main_process_only: null
+  ddp_find_unused_parameters: false
+  max_grad_norm: 1.0
+  trainer_kwargs:
+    max_seq_length: 16384
+    remove_unused_columns: false
+    dataset_kwargs:
+      skip_prepare_dataset: true
+  verl_config_overrides: {}
+  profiler:
+    save_dir: null
+    enable_cpu_profiling: false
+    enable_cuda_profiling: false
+    record_shapes: false
+    profile_memory: false
+    with_stack: false
+    with_flops: false
+    with_modules: false
+    row_limit: 50
+    schedule:
+      enable_schedule: false
+      wait: 0
+      warmup: 1
+      active: 3
+      repeat: 1
+      skip_first: 1
+  telemetry:
+    telemetry_dir: telemetry
+    collect_telemetry_for_all_ranks: false
+    track_gpu_temperature: false
+  empty_device_cache_steps: 1
+  nccl_default_timeout_minutes: null
+  label_ignore_index: null
+peft:
+  lora_r: 8
+  lora_alpha: 8
+  lora_dropout: 0.0
+  lora_target_modules: null
+  lora_modules_to_save: null
+  lora_bias: none
+  lora_init_weights: DEFAULT
+  lora_task_type: CAUSAL_LM
+  q_lora: false
+  q_lora_bits: 4
+  bnb_4bit_quant_type: fp4
+  llm_int8_skip_modules: null
+  use_bnb_nested_quant: false
+  bnb_4bit_quant_storage: uint8
+  bnb_4bit_compute_dtype: float32
+  peft_save_mode: ADAPTER_ONLY
+fsdp:
+  enable_fsdp: true
+  sharding_strategy: FULL_SHARD
+  cpu_offload: false
+  mixed_precision: null
+  backward_prefetch: BACKWARD_PRE
+  forward_prefetch: true
+  use_orig_params: true
+  state_dict_type: FULL_STATE_DICT
+  auto_wrap_policy: TRANSFORMER_BASED_WRAP
+  min_num_params: 100000
+  transformer_layer_cls: transformers.models.qwen2_5_vl.modeling_qwen2_5_vl.Qwen2_5_VLDecoderLayer
+  sync_module_states: true

telemetry/world_size.json

@@ -0,0 +1,4 @@
+{
+  "LOCAL_WORLD_SIZE": 8,
+  "WORLD_SIZE": 8
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

tokenizer_config.json

@@ -0,0 +1,210 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{%- for message in messages -%}{%- if loop.first and message['role'] != 'system' -%}{{ '<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n' }}{%- endif -%}{{ '<|im_start|>' + message['role'] + '\\n' }}{%- if message['content'] is string -%}{{- message['content'] -}}{%- elif message['content'] is iterable -%}{%- for item in message['content'] -%}{%- if item['type'].startswith('image') -%}{%- set image_count.value = image_count.value + 1 -%}{%- if add_vision_id -%}{{ 'Picture ' + image_count.value + ': ' }}{%- endif -%}{{ '<|vision_start|><|image_pad|><|vision_end|>' }}{%- elif item['type'].startswith('video') -%}{%- set video_count.value = video_count.value + 1 -%}{%- if add_vision_id -%}{{ 'Video ' + video_count.value + ': ' }}{%- endif -%}{{ '<|vision_start|><|video_pad|><|vision_end|>' }}{%- elif item['type']=='text' -%}{{- item['text'] if 'text' in item else item['content'] -}}{%- endif -%}{%- endfor -%}{%- endif -%}{{ '<|im_end|>\\n' }}{%- endfor -%}{%- if add_generation_prompt -%}{{- '<|im_start|>assistant\\n' -}}{%- endif -%}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 16384,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "processor_class": "Qwen2_5_VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,117 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 5.0,
+  "eval_steps": 500,
+  "global_step": 375,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 9.75,
+      "learning_rate": 1.9491669984901377e-05,
+      "loss": 0.6663,
+      "mean_token_accuracy": 0.8282047653198242,
+      "num_input_tokens_seen": 2327984,
+      "num_tokens": 2327984.0,
+      "step": 50
+    },
+    {
+      "epoch": 1.3333333333333333,
+      "grad_norm": 17.25,
+      "learning_rate": 1.72967916579403e-05,
+      "loss": 0.3542,
+      "mean_token_accuracy": 0.9049889421463013,
+      "num_input_tokens_seen": 5204816,
+      "num_tokens": 5204816.0,
+      "step": 100
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 3.25,
+      "learning_rate": 1.3756762552443555e-05,
+      "loss": 0.0478,
+      "mean_token_accuracy": 0.9875464010238647,
+      "num_input_tokens_seen": 7271696,
+      "num_tokens": 7271696.0,
+      "step": 150
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 2.171875,
+      "learning_rate": 9.524180841762577e-06,
+      "loss": 0.014,
+      "mean_token_accuracy": 0.9964023971557617,
+      "num_input_tokens_seen": 9599680,
+      "num_tokens": 9599680.0,
+      "step": 200
+    },
+    {
+      "epoch": 3.3333333333333335,
+      "grad_norm": 0.00946044921875,
+      "learning_rate": 5.379315560596038e-06,
+      "loss": 0.0023,
+      "mean_token_accuracy": 0.9995168459415436,
+      "num_input_tokens_seen": 12476512,
+      "num_tokens": 12476512.0,
+      "step": 250
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 0.169921875,
+      "learning_rate": 2.0862653732958914e-06,
+      "loss": 0.0002,
+      "mean_token_accuracy": 0.9999828958511352,
+      "num_input_tokens_seen": 14543392,
+      "num_tokens": 14543392.0,
+      "step": 300
+    },
+    {
+      "epoch": 4.666666666666667,
+      "grad_norm": 0.0380859375,
+      "learning_rate": 2.520983216615047e-07,
+      "loss": 0.0001,
+      "mean_token_accuracy": 1.0,
+      "num_input_tokens_seen": 16871376,
+      "num_tokens": 16871376.0,
+      "step": 350
+    },
+    {
+      "epoch": 5.0,
+      "mean_token_accuracy": 1.0,
+      "num_input_tokens_seen": 18179240,
+      "num_tokens": 18179240.0,
+      "step": 375,
+      "total_flos": 1.0562823256014848e+17,
+      "train_loss": 0.14466818872839213,
+      "train_runtime": 6016.1372,
+      "train_samples_per_second": 0.499,
+      "train_steps_per_second": 0.062,
+      "train_tokens_per_second": 176.962
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 375,
+  "num_input_tokens_seen": 18179240,
+  "num_train_epochs": 5,
+  "save_steps": 0,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.0562823256014848e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c8953db37e2e7554c2e72fff1c08c2f46abb56ee771d6021e97fde32145b7cb
+size 6097