Upload processor

preprocessor_config.json

@@ -1,6 +1,6 @@
 {
   "auto_map": {
-    "AutoProcessor": "processing_meralion.MERaLiONProcessor"
+    "AutoProcessor": "processing_meralion2.MERaLiON2Processor"
   },
   "chunk_length": 30,
   "dither": 0.0,
@@ -12,7 +12,7 @@
   "nb_max_frames": 3000,
   "padding_side": "right",
   "padding_value": 0.0,
-  "processor_class": "MERaLiONProcessor",
+  "processor_class": "MERaLiON2Processor",
   "return_attention_mask": false,
   "sampling_rate": 16000
 }

processing_meralion2.py

@@ -0,0 +1,194 @@
+"""Processor class for MERaLiON2."""
+
+from typing import List, Optional, Union
+
+import numpy as np
+
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.processing_utils import ProcessorMixin
+from transformers.tokenization_utils_base import PaddingStrategy, PreTokenizedInput, TextInput
+
+
+# copied from transformers.models.qwen2_audio.processing_qwen2_audio.Qwen2AudioProcessor
+class MERaLiON2Processor(ProcessorMixin):
+    r"""
+    Constructs a MERaLiON2 processor which wraps a whisper feature extractor and a gemma tokenizer into a single processor.
+
+    [`MERaLiON2Processor`] offers all the functionalities of [`WhisperFeatureExtractor`] and [`GemmaTokenizer`]. See the
+    [`~MERaLiON2Processor.__call__`] and [`~MERaLiON2Processor.decode`] for more information.
+
+    Args:
+        feature_extractor ([`WhisperFeatureExtractor`], *optional*):
+            The feature extractor is a required input.
+        tokenizer ([`GemmaTokenizer`], *optional*):
+            The tokenizer is a required input.
+        chat_template (`Optional[str]`, *optional*):
+                The Jinja template to use for formatting the conversation. If not provided, the default chat template
+                is used.
+    """
+
+    attributes = ["feature_extractor", "tokenizer"]
+    feature_extractor_class = "WhisperFeatureExtractor"
+    tokenizer_class = "AutoTokenizer"
+    valid_kwargs = [
+        "fixed_speech_embeds_length", 
+        "speech_token_index", 
+        "time_duration_limit", 
+        "whisper_chunk_size",
+        "do_normalize"
+    ]
+
+    def __init__(
+        self, 
+        feature_extractor=None, 
+        tokenizer=None, 
+        fixed_speech_embeds_length=100,
+        speech_token_index=255999,
+        time_duration_limit=300,
+        whisper_chunk_size=30,
+        do_normalize=True
+    ):
+        self.fixed_speech_embeds_length = fixed_speech_embeds_length
+        self.speech_token_index = speech_token_index
+        self.whisper_chunk_size = whisper_chunk_size
+        self.number_chunk_limit = time_duration_limit // whisper_chunk_size
+        self.do_normalize = do_normalize
+
+        super().__init__(feature_extractor, tokenizer)
+
+        self.speech_token = self.tokenizer.added_tokens_decoder[self.speech_token_index].content
+        self.feature_chunk_size = self.whisper_chunk_size * self.feature_extractor.sampling_rate
+
+    def _process_text(self, text: List[str], audio_number_chunks: np.ndarray):
+        pieces = []
+        for i, item in enumerate(text):
+            target_string = self.speech_token * self.fixed_speech_embeds_length * audio_number_chunks[i]
+            pieces.append(item.replace(self.speech_token, target_string))
+        return pieces
+
+    def _get_number_chunks(self, audios: List[np.ndarray]):
+        audio_lengths = np.array([_.shape[0] for _ in audios])
+        number_chunks = (audio_lengths // self.feature_chunk_size) + 1
+        return np.clip(number_chunks, a_min=None, a_max=self.number_chunk_limit)
+
+    def _get_chunked_audios(self, audios: Union[np.ndarray, List[np.ndarray]]):
+        if isinstance(audios, np.ndarray):
+            audios = [audios]
+
+        audio_number_chunks = self._get_number_chunks(audios)
+        chunked_audios = []
+
+        for audio_idx, audio in enumerate(audios):
+            for cid in range(audio_number_chunks[audio_idx]):
+                chunked_audios.append(
+                    audio[cid * self.feature_chunk_size: (cid + 1) * self.feature_chunk_size]
+                )
+        return audio_number_chunks, chunked_audios
+
+    def __call__(
+        self,
+        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
+        audios: Union[np.ndarray, List[np.ndarray]] = None,
+        padding: Union[bool, str, PaddingStrategy] = True,
+        sampling_rate: Optional[int] = None,
+        do_normalize: Optional[bool] = None,
+        **kwargs,
+    ) -> BatchFeature:
+        """
+        Main method to prepare for the model one or several sequences(s) and audio(s). This method forwards the `text`
+        and `kwargs` arguments to GemmaTokenizer's [`~GemmaTokenizer.__call__`] if `text` is not `None` to encode
+        the text. To prepare the audio(s), this method forwards the `audios` and `kwrags` arguments to
+        WhisperFeatureExtractor's [`~WhisperFeatureExtractor.__call__`] if `audios` is not `None`. Please refer to the doctsring
+        of the above two methods for more information.
+
+        Args:
+            text (`str`, `List[str]`):
+                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
+                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
+                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
+            audios (`np.ndarray`, `List[np.ndarray]`):
+                The audio or batch of audios to be prepared. Each audio can be a NumPy array.
+            padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
+                Select a strategy to pad the returned sequences (according to the model's padding side and padding
+                index) among:
+                - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+                  sequence if provided).
+                - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+                  acceptable input length for the model if that argument is not provided.
+                - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+                  lengths).
+            sampling_rate (`int`, defaults to 16000):
+                The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
+            do_normalize (`bool`, defaults to `True`):
+                Whether or not to zero-mean unit-variance normalize the input. 
+                Normalizing can help to significantly improve the performance of the model.
+        """
+
+        if text is None:
+            raise ValueError("You need to specify either a `text` input to process.")
+        if not isinstance(text, list):
+            text = [text]
+        if not isinstance(audios, list):
+            audios = [audios]
+        if sampling_rate is None:
+            sampling_rate = self.feature_extractor.sampling_rate
+        if do_normalize is None:
+            do_normalize = self.do_normalize
+
+        for i, audio in enumerate(audios):
+            if audio.ndim > 1:
+                raise Exception(f"MERaLiON2 only accepts mono channel audio, {i+1}th audio have {audios[0].ndim} channels")
+        
+        inputs_dict = {}
+        
+        if audios is not None:
+            audio_number_chunks, chunked_audios = self._get_chunked_audios(audios)
+            text = self._process_text(text, audio_number_chunks)
+            
+            audio_inputs = self.feature_extractor(
+                chunked_audios, 
+                sampling_rate=sampling_rate, 
+                return_tensors="pt",
+                return_attention_mask=True, 
+                padding="max_length", 
+                do_normalize=self.do_normalize,
+                **kwargs
+            )
+            audio_inputs["feature_attention_mask"] = audio_inputs.pop(
+                "attention_mask"
+            )  # rename attention_mask to prevent conflicts later on
+            inputs_dict.update(audio_inputs)
+
+        text_input = self.tokenizer(
+            text=text,
+            return_tensors="pt",
+            add_special_tokens=False,
+            return_attention_mask=True,
+            padding=padding,
+            **kwargs
+        )   
+
+        inputs_dict["input_ids"] = text_input.input_ids
+        inputs_dict["attention_mask"] = text_input.attention_mask
+
+        return BatchFeature(data={**inputs_dict})
+
+    def batch_decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to GemmaTokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please
+        refer to the docstring of this method for more information.
+        """
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    def decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to GemmaTokenizer's [`~PreTrainedTokenizer.decode`]. Please refer to
+        the docstring of this method for more information.
+        """
+        return self.tokenizer.decode(*args, **kwargs)
+
+    @property
+    def model_input_names(self):
+        tokenizer_input_names = self.tokenizer.model_input_names
+        feature_extractor_input_names = self.feature_extractor.model_input_names
+        return list(dict.fromkeys(tokenizer_input_names + feature_extractor_input_names + ["feature_attention_mask"]))

processor_config.json

@@ -1,10 +1,10 @@
 {
   "auto_map": {
-    "AutoProcessor": "processing_meralion.MERaLiONProcessor"
+    "AutoProcessor": "processing_meralion2.MERaLiON2Processor"
   },
   "do_normalize": true,
   "fixed_speech_embeds_length": 100,
-  "processor_class": "MERaLiONProcessor",
+  "processor_class": "MERaLiON2Processor",
   "speech_token_index": 255999,
   "whisper_chunk_size": 30
 }

tokenizer_config.json

@@ -2000,7 +2000,7 @@
     "<end_of_turn>"
   ],
   "auto_map": {
-    "AutoProcessor": "processing_meralion.MERaLiONProcessor"
+    "AutoProcessor": "processing_meralion2.MERaLiON2Processor"
   },
   "bos_token": "<bos>",
   "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
@@ -2010,7 +2010,7 @@
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<pad>",
   "padding_side": "left",
-  "processor_class": "MERaLiONProcessor",
+  "processor_class": "MERaLiON2Processor",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "GemmaTokenizer",