step2000/config.json

{"model": {"d_model": 512, "vocab_size": 100352, "n_layers": 16, "block": {"sequence_mixer": {"name": "default", "n_heads": 16, "n_kv_heads": 4, "head_dim": 32, "bias": false, "rope": {"name": "default", "theta": 500000, "full_precision": true, "no_global_rope": false, "partial_rotary_factor": 1.0, "_CLASS_": "olmo_core.nn.rope.RoPEConfig"}, "qk_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "backend": "flash_4", "dtype": "float32", "sliding_window": {"pattern": [512, 512, -1], "force_full_attention_on_first_layer": false, "force_full_attention_on_last_layer": true, "_CLASS_": "olmo_core.nn.attention.SlidingWindowAttentionConfig"}, "use_head_qk_norm": false, "_CLASS_": "olmo_core.nn.attention.AttentionConfig", "type": "attention"}, "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "feed_forward": {"hidden_size": 3584, "name": "default", "bias": false, "dtype": "float32", "activation": "silu", "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig"}, "name": "reordered_norm", "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig"}, "lm_head": {"name": "default", "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "bias": false, "dtype": "float32", "loss_implementation": "default", "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig"}, "name": "default", "dtype": "float32", "init_method": "normal", "init_seed": 0, "init_std": 0.02, "tie_word_embeddings": true, "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig"}, "dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "mix": "OLMo-midtraining-mix-0625-25B", "mix_base_dir": "https://olmo-data.org", "expand_glob": false, "include_instance_metadata": true, "instance_filter_config": {"repetition_max_period": 13, "repetition_min_period": 1, "repetition_max_count": 32, "_CLASS_": "olmo_core.data.numpy_dataset.InstanceFilterConfig"}, "work_dir": "/root/saves-mt", "ignore_fingerprint_mismatch": false, "sequence_length": 8192, "max_target_sequence_length": 8192, "generate_doc_lengths": false, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyFSLDatasetConfig"}, "data_loader": {"global_batch_size": 1048576, "seed": 34521, "num_workers": 8, "prefetch_factor": 2048, "ignore_fingerprint_mismatch": false, "_CLASS_": "olmo_core.data.data_loader.NumpyDataLoaderConfig", "type": "numpy"}, "train_module": {"rank_microbatch_size": 131072, "max_sequence_length": 8192, "optim": {"group_overrides": [{"params": ["embeddings.weight"], "opts": {"weight_decay": 0.0}, "_CLASS_": "olmo_core.optim.config.OptimGroupOverride"}], "compile": false, "fixed_fields": ["initial_lr"], "lr": 0.0002, "betas": [0.9, 0.95], "eps": 1e-08, "weight_decay": 0.1, "foreach": true, "step_increment_bugfix": true, "rolling_interval_length": 128, "sigma_factor": 6, "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig", "type": "skip_step_adamw"}, "max_grad_norm": 1.0, "scheduler": {"lr_field": "lr", "initial_lr_field": "initial_lr", "units": "steps", "warmup": 500, "alpha_f": 0.1, "warmup_min_lr": 0.0, "_CLASS_": "olmo_core.optim.scheduler.CosWithWarmup", "type": "cos_with_warmup"}, "compile_model": true, "float8_config": {"enabled": false, "_CLASS_": "olmo_core.float8.Float8Config"}, "dp_config": {"name": "ddp", "param_dtype": "bfloat16", "reduce_dtype": "float32", "wrapping_strategy": "full", "prefetch_factor": 0, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig"}, "ac_config": {"mode": "budget", "activation_memory_budget": 0.75, "determinism_check": "default", "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerActivationCheckpointingConfig"}, "z_loss_multiplier": 1e-05, "label_ignore_index": -100, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerTrainModuleConfig"}, "trainer": {"save_folder": "/root/saves-mt", "work_dir": "/root/saves-mt", "load_path": "/root/saves/step75073", "load_strategy": "always", "load_optim_state": true, "load_trainer_state": false, "checkpointer": {"pre_download": false, "throttle_uploads": false, "_CLASS_": "olmo_core.train.checkpoint.CheckpointerConfig"}, "save_overwrite": true, "max_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "cancel_check_interval": 10, "metrics_collect_interval": 50, "callbacks": {"config_saver": {"fname": "config.json", "_CLASS_": "olmo_core.train.callbacks.config_saver.ConfigSaverCallback"}, "checkpointer": {"save_interval": 1000, "save_async": false, "remove": "ephemeral_only", "max_checkpoints": 3, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.checkpointer.CheckpointerCallback"}, "comet": {"enabled": false, "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "notifications": "none", "failure_tag": "failed", "auto_resume": false, "_CLASS_": "olmo_core.train.callbacks.comet.CometCallback"}, "wandb": {"enabled": true, "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "_CLASS_": "olmo_core.train.callbacks.wandb.WandBCallback"}, "lm_evaluator": {"eval_dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "mix": "v3-small-ppl-validation", "mix_base_dir": "https://olmo-data.org", "expand_glob": false, "include_instance_metadata": true, "work_dir": "/root/saves-mt", "ignore_fingerprint_mismatch": false, "sequence_length": 8192, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyPaddedFSLDatasetConfig"}, "eval_interval": 1000, "eval_on_startup": false, "eval_on_finish": false, "cancel_after_first_eval": false, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "log_interval": 5, "deterministic": true, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.LMEvaluatorCallbackConfig"}, "downstream_evaluator": {"tasks": ["arc_challenge_test_mc_5shot_fast", "arc_easy_test_mc_5shot_fast", "hellaswag_bpb_5shot"], "tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "eval_interval": 1000, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "eval_on_startup": false, "eval_on_finish": false, "cancel_after_first_eval": false, "log_interval": 5, "lazy": false, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.DownstreamEvaluatorCallbackConfig"}}, "bookkeeping_soft_timeout": 30, "no_checkpoints": false, "no_evals": false, "_CLASS_": "olmo_core.train.config.TrainerConfig"}, "init_seed": 12536, "_CLASS_": "olmo_core.script_utils.ExperimentConfig"}