| | from transformers.models.qwen2.configuration_qwen2 import Qwen2Config |
| |
|
| |
|
| | class KimiAudioConfig(Qwen2Config): |
| | def __init__( |
| | self, |
| | vocab_size=163840, |
| | hidden_size=4096, |
| | intermediate_size=11008, |
| | num_hidden_layers=32, |
| | num_attention_heads=32, |
| | num_key_value_heads=None, |
| | hidden_act="silu", |
| | initializer_range=0.02, |
| | rms_norm_eps=1e-6, |
| | use_cache=True, |
| | rope_theta=10000.0, |
| | rope_scaling=None, |
| | tie_word_embeddings=False, |
| | kimia_mimo_layers: int = 6, |
| | kimia_mimo_audiodelaytokens: int = 5, |
| | kimia_mimo_transformer_from_layer_index: int = 21, |
| | kimia_audio_output_vocab: int = 16896, |
| | kimia_text_output_vocab: int = 152064, |
| | num_audio_special_tokens: int = 512, |
| | num_base_tokens: int = 151643, |
| | kimia_token_offset: int = 152064, |
| | use_whisper_feature: bool = True, |
| | kimia_adaptor_input_dim: int = 5120, |
| | kimia_media_begin: int = 151661, |
| | kimia_media_end: int = 151663, |
| | **kwargs, |
| | ): |
| | super().__init__( |
| | vocab_size=vocab_size, |
| | hidden_size=hidden_size, |
| | intermediate_size=intermediate_size, |
| | num_hidden_layers=num_hidden_layers, |
| | num_attention_heads=num_attention_heads, |
| | num_key_value_heads=num_key_value_heads, |
| | hidden_act=hidden_act, |
| | initializer_range=initializer_range, |
| | rms_norm_eps=rms_norm_eps, |
| | use_cache=use_cache, |
| | tie_word_embeddings=tie_word_embeddings, |
| | rope_theta=rope_theta, |
| | rope_scaling=rope_scaling, |
| | **kwargs, |
| | ) |
| |
|
| | self.kimia_mimo_layers = kimia_mimo_layers |
| | self.kimia_mimo_audiodelaytokens = kimia_mimo_audiodelaytokens |
| | |
| | self.kimia_mimo_transformer_from_layer_index = ( |
| | kimia_mimo_transformer_from_layer_index |
| | ) |
| | self.kimia_audio_output_vocab = kimia_audio_output_vocab |
| | self.kimia_text_output_vocab = kimia_text_output_vocab |
| | self.num_audio_special_tokens = num_audio_special_tokens |
| | self.num_base_tokens = num_base_tokens |
| | self.kimia_token_offset = kimia_token_offset |
| | self.use_whisper_feature = use_whisper_feature |
| | self.kimia_adaptor_input_dim = kimia_adaptor_input_dim |
| | |
| | self.kimia_media_begin = kimia_media_begin |
| | self.kimia_media_end = kimia_media_end |
| |
|
