Upload folder using huggingface_hub

audio_processing_hyperclovax_omni.py

@@ -1,10 +1,21 @@
-import torch
+"""
+HyperCLOVAX Omni Audio Processor
+
+Implements Whisper-compatible audio feature extraction:
+- Log-mel spectrogram extraction from waveform
+- Chunked processing for long audio clips
+- Attention mask generation for padded sequences
+- Discrete audio token count calculation (conv-based)
+"""
+
+from typing import List, Optional
+
 import numpy as np
-from typing import List, Union, Optional
+import torch
 from transformers import BatchFeature
-from transformers.processing_utils import AudioKwargs
+from transformers.audio_utils import mel_filter_bank
 from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor
-from transformers.audio_utils import mel_filter_bank, spectrogram, window_function
+from transformers.processing_utils import AudioKwargs
 
 
 class HyperCLOVAXOmniAudioKwargs(AudioKwargs, total=False):
@@ -29,6 +40,13 @@ class HyperCLOVAXOmniAudioKwargs(AudioKwargs, total=False):
 
 
 class HyperCLOVAXOmniAudioProcessor(SequenceFeatureExtractor):
+    """Audio processor for HyperCLOVAX Omni.
+
+    Extracts Whisper-compatible log-mel spectrogram features and computes
+    attention masks for the audio encoder. Also supports discrete audio
+    token count calculation.
+    """
+
     model_input_names = ["audio_values", "audio_masks", "discrete_audio_values"]
 
     def __init__(
@@ -93,17 +111,21 @@ class HyperCLOVAXOmniAudioProcessor(SequenceFeatureExtractor):
             mel_scale="slaney",
         )
 
-    def _extract_fbank_features(self, waveform_batch: np.ndarray, device: str = "cpu") -> np.ndarray:
-        """Waveform 배치에서 log-mel spectrogram 특징을 추출합니다.
+    def _extract_fbank_features(
+        self,
+        waveform_batch: np.ndarray,
+        device: str = "cpu",
+    ) -> np.ndarray:
+        """Extract log-mel spectrogram features from a waveform batch.
 
-        WhisperFeatureExtractor._torch_extract_fbank_features와 동일한 로직입니다.
+        Follows the same logic as WhisperFeatureExtractor._torch_extract_fbank_features.
 
         Args:
-            waveform_batch: (batch_size, n_samples) shape의 waveform 배열.
-            device: 연산에 사용할 디바이스. 기본 "cpu".
+            waveform_batch: Waveform array of shape (batch_size, n_samples).
+            device: Device for computation. Defaults to "cpu".
 
         Returns:
-            (batch_size, feature_size, num_frames) shape의 log-mel spectrogram.
+            Log-mel spectrogram of shape (batch_size, feature_size, num_frames).
         """
         waveform = torch.from_numpy(waveform_batch).to(device, torch.float32)
         window = torch.hann_window(self.n_fft, device=device)
@@ -135,20 +157,19 @@ class HyperCLOVAXOmniAudioProcessor(SequenceFeatureExtractor):
         chunks: List[np.ndarray],
         sampling_rate: int,
     ) -> dict:
-        """오디오 청크 리스트를 패딩하고 mel-spectrogram을 추출합니다.
+        """Pad audio chunks and extract mel-spectrogram features.
 
-        각 청크를 n_samples 길이로 패딩한 뒤, mel-spectrogram을 추출하고
-        attention mask를 생성합니다. WhisperFeatureExtractor.__call__의
-        핵심 로직을 대체합니다.
+        Each chunk is padded to n_samples length, then mel-spectrogram is
+        extracted and an attention mask is generated.
 
         Args:
-            chunks: 1D numpy array 리스트. 각 배열은 하나의 오디오 청크.
-            sampling_rate: 오디오 샘플링 레이트.
+            chunks: List of 1D numpy arrays, each representing an audio chunk.
+            sampling_rate: Audio sampling rate.
 
         Returns:
-            dict with:
-                - "input_features": (num_chunks, feature_size, nb_max_frames) numpy 배열.
-                - "attention_mask": (num_chunks, nb_max_frames) numpy 배열.
+            Dictionary with:
+                - "input_features": Array of shape (num_chunks, feature_size, nb_max_frames).
+                - "attention_mask": Array of shape (num_chunks, nb_max_frames).
         """
         n_samples = self.chunk_length * sampling_rate
         nb_max_frames = n_samples // self.hop_length
@@ -160,7 +181,7 @@ class HyperCLOVAXOmniAudioProcessor(SequenceFeatureExtractor):
             chunk = np.asarray(chunk, dtype=np.float32)
             chunk_len = len(chunk)
 
-            # 패딩 또는 트렁케이션
+            # Pad or truncate
             if chunk_len < n_samples:
                 padded = np.full(n_samples, self.padding_value, dtype=np.float32)
                 padded[:chunk_len] = chunk
@@ -170,11 +191,10 @@ class HyperCLOVAXOmniAudioProcessor(SequenceFeatureExtractor):
 
             padded_waveforms.append(padded)
 
-            # Attention mask (sample-level → frame-level)
+            # Attention mask (sample-level -> frame-level)
             sample_mask = np.zeros(n_samples, dtype=np.int32)
             sample_mask[:chunk_len] = 1
             frame_mask = sample_mask[:: self.hop_length]
-            # nb_max_frames 길이로 맞춤
             if len(frame_mask) > nb_max_frames:
                 frame_mask = frame_mask[:nb_max_frames]
             elif len(frame_mask) < nb_max_frames:
@@ -190,75 +210,32 @@ class HyperCLOVAXOmniAudioProcessor(SequenceFeatureExtractor):
             "attention_mask": attention_mask,
         }
 
-    def get_num_audio_tokens(
-        self,
-        audio_masks: "torch.Tensor",
-        discrete_audio_values: Optional["torch.Tensor"] = None,
-        include_boundary_tokens: bool = False,
-        chunk_unit: Optional[int] = None,
-        sampling_rate: Optional[int] = None,
-    ) -> tuple:
-        """오디오 입력에 대한 (continuous, discrete) 토큰 수를 계산합니다.
+    def _get_feature_lengths(self, audio_masks: torch.Tensor) -> torch.Tensor:
+        """Compute feature lengths after conv downsampling.
 
         Args:
-            audio_masks: continuous audio의 attention mask. (N,) 또는 (num_chunks, N).
-            discrete_audio_values: discrete audio waveform. None이면 discrete 계산 생략.
-            include_boundary_tokens: start/end 토큰 포함 여부.
-            chunk_unit: discrete 처리 시 청크 단위 (초). None이면 self.chunk_unit 사용.
-            sampling_rate: 샘플링 레이트. None이면 self.sampling_rate 사용.
+            audio_masks: Attention mask of shape (batch, nb_max_frames).
 
         Returns:
-            (num_continuous_tokens, num_discrete_tokens) 튜플.
-            discrete 미사용 시 num_discrete_tokens는 0.
+            Feature lengths tensor of shape (batch,).
         """
-        chunk_unit = chunk_unit if chunk_unit is not None else self.chunk_unit
-        sampling_rate = sampling_rate if sampling_rate is not None else self.sampling_rate
-
-        def _compute_continuous_tokens(audio_mask: "torch.Tensor") -> int:
-            input_length = (int(audio_mask.sum()) - 1) // 2 + 1
-            return (input_length - 2) // 2 + 1
-
-        num_continuous_tokens, num_discrete_tokens = 0, 0
-        if len(audio_masks.shape) == 1:
-            num_continuous_tokens = _compute_continuous_tokens(audio_masks)
-        else:
-            num_continuous_tokens = sum(_compute_continuous_tokens(m) for m in audio_masks)
-        if include_boundary_tokens:
-            num_continuous_tokens += 2  # audio_start_token, audio_end_token
+        return (audio_masks.sum(-1) - 1) // 2 + 1
 
-        if (
-            self.use_discrete_audio_token
-            and discrete_audio_values is not None
-        ):
-            audio_length = len(discrete_audio_values)
-            chunk_size = chunk_unit * sampling_rate
-            for _start in range(0, audio_length, chunk_size):
-                _end = min(_start + chunk_size, audio_length)
-                _chunked_length = _end - _start
-                mel_len = _chunked_length // 160
-                after_conv1 = (mel_len + 2 * 1 - 1 * (3 - 1) - 1) // 2 + 1
-                code_len = (after_conv1 + 2 * 1 - 1 * (3 - 1) - 1) // 2 + 1
-                num_discrete_tokens += code_len
-            if include_boundary_tokens:
-                num_discrete_tokens += 2  # discrete_audio_start_token, discrete_audio_end_token
+    def _get_attention_mask(self, audio_masks: torch.Tensor) -> torch.Tensor:
+        """Generate attention mask for the audio encoder.
 
-        return (num_continuous_tokens, num_discrete_tokens)
+        Creates a causal-style mask where padded positions are filled with -inf.
 
-    def _get_feature_lengths(
-        self,
-        audio_masks: torch.Tensor,
-    ):
-        return (audio_masks.sum(-1) - 1) // 2 + 1
+        Args:
+            audio_masks: Attention mask of shape (batch, nb_max_frames).
 
-    def _get_attention_mask(
-        self,
-        audio_values: torch.Tensor,
-        audio_masks: torch.Tensor,
-    ):
+        Returns:
+            Attention mask of shape (batch, 1, max_seq_len, max_seq_len).
+        """
         feature_lengths = self._get_feature_lengths(audio_masks=audio_masks)
-        max_seq_len = (self.nb_max_frames - 2) // 2 + 1 
+        max_seq_len = (self.nb_max_frames - 2) // 2 + 1
         padding_mask = torch.arange(max_seq_len) >= feature_lengths.unsqueeze(1)
-        attention_mask = padding_mask[:, None, None, :].expand(len(audio_values), 1, max_seq_len, max_seq_len)
+        attention_mask = padding_mask[:, None, None, :].expand(padding_mask.shape[0], 1, max_seq_len, max_seq_len)
         attention_mask = attention_mask.masked_fill(attention_mask, float("-inf"))
         return attention_mask
 
@@ -268,35 +245,32 @@ class HyperCLOVAXOmniAudioProcessor(SequenceFeatureExtractor):
         sampling_rate: Optional[int] = None,
         chunk_length: Optional[int] = None,
     ) -> dict:
-        """오디오 클립들에 대한 continuous audio 전처리를 수행합니다.
+        """Preprocess audio clips for continuous audio features.
 
-        각 오디오 클립을 chunk_length(초) 단위로 분할한 뒤 mel-spectrogram
-        특징을 추출하고, attention mask 기반으로 토큰 수를 계산합니다.
+        Splits each audio clip into chunks of chunk_length seconds, extracts
+        mel-spectrogram features, and computes token counts from attention masks.
 
         Args:
-            audio_clips: 오디오 클립 리스트.
-                각 클립은 1D numpy array (mono, float32).
-            sampling_rate: 오디오 샘플링 레이트. None이면 self.sampling_rate 사용.
-            chunk_length: 오디오를 분할할 단위 (초). None이면 self.chunk_length 사용.
+            audio_clips: List of audio clips, each a 1D numpy array (mono, float32).
+            sampling_rate: Audio sampling rate. Defaults to self.sampling_rate.
+            chunk_length: Chunk duration in seconds. Defaults to self.chunk_length.
 
         Returns:
-            dict with:
-                - "audio_values": (num_total_chunks, feature_size, nb_max_frames) 텐서.
-                - "audio_masks": (num_total_chunks, nb_max_frames) 텐서.
-                - "num_audio_tokens": (N,) 텐서. 클립별 continuous 토큰 수.
+            Dictionary with:
+                - "audio_values": Tensor of shape (num_total_chunks, feature_size, nb_max_frames).
+                - "audio_masks": Tensor of shape (num_total_chunks, nb_max_frames).
+                - "audio_attention_mask": Tensor of shape (num_total_chunks, max_seq_len, max_seq_len).
+                - "num_audio_tokens": Tensor of shape (N,) with per-clip continuous token counts.
         """
         sampling_rate = sampling_rate if sampling_rate is not None else self.sampling_rate
         chunk_length = chunk_length if chunk_length is not None else self.chunk_length
 
         if len(audio_clips) == 0:
-            _audio_values = torch.zeros(0, self.feature_size, self.nb_max_frames)
-            _audio_masks = torch.zeros(0, self.nb_max_frames)
             max_seq_len = (self.nb_max_frames - 2) // 2 + 1
-            _audio_attention_mask = torch.zeros(0, 1, max_seq_len, max_seq_len)
             return {
-                "audio_values": _audio_values,
-                "_audio_attention_mask": _audio_attention_mask,
-                "audio_masks": _audio_masks,
+                "audio_values": torch.zeros(0, self.feature_size, self.nb_max_frames),
+                "audio_masks": torch.zeros(0, self.nb_max_frames),
+                "audio_attention_mask": torch.zeros(0, max_seq_len, max_seq_len),
                 "num_audio_tokens": torch.tensor([], dtype=torch.long),
             }
 
@@ -311,21 +285,19 @@ class HyperCLOVAXOmniAudioProcessor(SequenceFeatureExtractor):
 
             _audio_value = result["input_features"]
             _audio_mask = result["attention_mask"]
-
-            input_lengths = int(_audio_mask.sum())
-            input_lengths = (input_lengths - 1) // 2 + 1
-            output_lengths = (input_lengths - 2) // 2 + 1
+            _num_audio_token = 0
+            for _mask in _audio_mask:
+                _input_length = (_mask.shape[-1] - 1) // 2 + 1
+                _num_audio_token += (_input_length - 2) // 2 + 1
 
             _audio_values.append(torch.Tensor(_audio_value))
             _audio_masks.append(torch.Tensor(_audio_mask))
-            _num_audio_tokens.append(output_lengths)
+            _num_audio_tokens.append(_num_audio_token)
 
         _audio_values = torch.cat(_audio_values, dim=0)
         _audio_masks = torch.cat(_audio_masks, dim=0)
-        _audio_attention_mask = self._get_attention_mask(
-            audio_values=_audio_values,
-            audio_masks=_audio_masks,
-        )
+        _audio_attention_mask = self._get_attention_mask(audio_masks=_audio_masks)
+
         return {
             "audio_values": _audio_values,
             "audio_masks": _audio_masks,
@@ -340,23 +312,21 @@ class HyperCLOVAXOmniAudioProcessor(SequenceFeatureExtractor):
         chunk_unit: Optional[int] = None,
         min_chunk_size: Optional[int] = None,
     ) -> dict:
-        """오디오 클립들에 대한 discrete audio 전처리를 수행합니다.
+        """Preprocess audio clips for discrete audio tokens.
 
-        각 오디오 클립의 길이를 검증한 뒤, conv 레이어 기반으로
-        discrete token 수를 계산합니다. 오디오 원본 waveform은
-        패딩하여 텐서로 반환합니다.
+        Validates each audio clip and computes the number of discrete tokens
+        based on conv layer downsampling. Returns padded waveform tensors.
 
         Args:
-            audio_clips: 오디오 클립 리스트.
-                각 클립은 1D numpy array (mono, float32).
-            sampling_rate: 오디오 샘플링 레이트. None이면 self.sampling_rate 사용.
-            chunk_unit: 긴 오디오를 분할할 단위 (초). None이면 self.chunk_unit 사용.
-            min_chunk_size: 최소 오디오 길이 (샘플 수). None이면 self.min_chunk_size 사용.
+            audio_clips: List of audio clips, each a 1D numpy array (mono, float32).
+            sampling_rate: Audio sampling rate. Defaults to self.sampling_rate.
+            chunk_unit: Chunk duration in seconds for long audio. Defaults to self.chunk_unit.
+            min_chunk_size: Minimum audio length in samples. Defaults to self.min_chunk_size.
 
         Returns:
-            dict with:
-                - "discrete_audio_values": (N, max_audio_len) 텐서. 패딩된 waveform.
-                - "num_discrete_audio_tokens": (N,) 텐서. 클립별 discrete 토큰 수.
+            Dictionary with:
+                - "discrete_audio_values": Tensor of shape (N, max_audio_len).
+                - "num_discrete_audio_tokens": Tensor of shape (N,) with per-clip discrete token counts.
         """
         sampling_rate = sampling_rate if sampling_rate is not None else self.sampling_rate
         chunk_unit = chunk_unit if chunk_unit is not None else self.chunk_unit
@@ -419,26 +389,29 @@ class HyperCLOVAXOmniAudioProcessor(SequenceFeatureExtractor):
         min_chunk_size: Optional[int] = None,
         **kwargs,
     ) -> BatchFeature:
-        """오디오 클립들에 대한 전처리를 수행합니다.
+        """Preprocess a list of audio clips.
 
-        오디오 클립 리스트에 대해 continuous audio 처리를 수행하고,
-        use_discrete_audio_token이 True인 경우 discrete audio 처리도 추가로
-        수행합니다.
+        Performs continuous audio processing on all clips. If use_discrete_audio_token
+        is enabled, discrete audio processing is also performed.
 
         Args:
-            audios: 오디오 클립 리스트. 각 원소는 1D numpy array.
-            sampling_rate: 오디오 샘플링 레이트. None이면 self.sampling_rate 사용.
-            chunk_length: continuous 처리 시 오디오 분할 단위 (초). None이면 기본값 사용.
-            chunk_unit: discrete 처리 시 오디오 분할 단위 (초). None이면 self.chunk_unit 사용.
-            min_chunk_size: discrete 처리 시 최소 오디오 길이 (샘플 수). None이면 self.min_chunk_size 사용.
+            audios: List of audio clips, each a 1D numpy array.
+            sampling_rate: Audio sampling rate. Defaults to self.sampling_rate.
+            chunk_length: Chunk duration in seconds for continuous processing.
+                Defaults to self.chunk_length.
+            chunk_unit: Chunk duration in seconds for discrete processing.
+                Defaults to self.chunk_unit.
+            min_chunk_size: Minimum audio length in samples for discrete processing.
+                Defaults to self.min_chunk_size.
 
         Returns:
             BatchFeature with:
-                - audio_values: (num_total_chunks, feature_size, nb_max_frames) 텐서.
-                - audio_masks: (num_total_chunks, nb_max_frames) 텐서.
-                - num_audio_tokens: (N,) 텐서. 클립별 continuous 토큰 수.
-                - discrete_audio_values (optional): (N, max_audio_len) 텐서.
-                - num_discrete_audio_tokens (optional): (N,) 텐서. 클립별 discrete 토큰 수.
+                - audio_values: Tensor of shape (num_total_chunks, feature_size, nb_max_frames).
+                - audio_masks: Tensor of shape (num_total_chunks, nb_max_frames).
+                - audio_attention_mask: Tensor of shape (num_total_chunks, max_seq_len, max_seq_len).
+                - num_audio_tokens: Tensor of shape (N,) with per-clip continuous token counts.
+                - discrete_audio_values (optional): Tensor of shape (N, max_audio_len).
+                - num_discrete_audio_tokens (optional): Tensor of shape (N,) with per-clip discrete token counts.
         """
         continuous_result = self._preprocess_continuous_audio(
             audios,
@@ -464,3 +437,60 @@ class HyperCLOVAXOmniAudioProcessor(SequenceFeatureExtractor):
             data["num_discrete_audio_tokens"] = discrete_result["num_discrete_audio_tokens"]
 
         return BatchFeature(data=data, tensor_type="pt")
+
+    def get_num_audio_tokens(
+        self,
+        audio_masks: torch.Tensor,
+        discrete_audio_values: Optional[torch.Tensor] = None,
+        include_boundary_tokens: bool = False,
+        chunk_unit: Optional[int] = None,
+        sampling_rate: Optional[int] = None,
+        return_tuple: Optional[bool] = None,
+    ) -> int:
+        """Compute the number of audio tokens for the given input.
+
+        Args:
+            audio_masks: Attention mask for continuous audio. Shape (N,) or (num_chunks, N).
+            discrete_audio_values: Discrete audio waveform. None to skip discrete computation.
+            include_boundary_tokens: Whether to include start/end boundary tokens.
+            chunk_unit: Chunk duration in seconds for discrete processing.
+                Defaults to self.chunk_unit.
+            sampling_rate: Audio sampling rate. Defaults to self.sampling_rate.
+            return_tuple: If True, return (continuous, discrete) tuple.
+                Otherwise return the sum.
+
+        Returns:
+            Token count as int, or (continuous, discrete) tuple if return_tuple is True.
+        """
+        chunk_unit = chunk_unit if chunk_unit is not None else self.chunk_unit
+        sampling_rate = sampling_rate if sampling_rate is not None else self.sampling_rate
+
+        def _compute_continuous_tokens(audio_mask: torch.Tensor) -> int:
+            input_length = (audio_mask.shape[-1] - 1) // 2 + 1
+            return (input_length - 2) // 2 + 1
+
+        num_continuous_tokens, num_discrete_tokens = 0, 0
+        if len(audio_masks.shape) == 1:
+            num_continuous_tokens = _compute_continuous_tokens(audio_masks)
+        else:
+            num_continuous_tokens = sum(_compute_continuous_tokens(m) for m in audio_masks)
+        if include_boundary_tokens:
+            num_continuous_tokens += 2
+
+        if self.use_discrete_audio_token and discrete_audio_values is not None:
+            audio_length = len(discrete_audio_values)
+            chunk_size = chunk_unit * sampling_rate
+            for _start in range(0, audio_length, chunk_size):
+                _end = min(_start + chunk_size, audio_length)
+                _chunked_length = _end - _start
+                mel_len = _chunked_length // 160
+                after_conv1 = (mel_len + 2 * 1 - 1 * (3 - 1) - 1) // 2 + 1
+                code_len = (after_conv1 + 2 * 1 - 1 * (3 - 1) - 1) // 2 + 1
+                num_discrete_tokens += code_len
+            if include_boundary_tokens:
+                num_discrete_tokens += 2
+
+        if return_tuple:
+            return (num_continuous_tokens, num_discrete_tokens)
+        else:
+            return num_continuous_tokens + num_discrete_tokens

chat_template.jinja

@@ -78,7 +78,7 @@
                     {%- set image_id = 'image_%02d' % ns_img.count %}
                     {%- set ns_img.count = ns_img.count + 1 %}
                     {{- '<|mime_start|>{"id": "' + image_id + '", "type": "image/jpeg", "filename": "' + content.get('filename', "a.jpg") + '"}<|mime_end|>\n' }}
-                    {{- '<|discrete_image_start|><|DISCRETE_IMAGE_PAD|><|discrete_image_end|>'}}
+                    {{- '<|discrete_image_start|><|DISCRETE_IMAGE_PAD|><|discrete_image_end|>\n'}}
                     {{- '<|image_start|><|IMAGE_PAD|><|image_end|>' }}
                 {%- elif content['type'] == 'video' or 'video' in content or 'video_url' in content %}
                     {%- set video_id = 'video_%02d' % ns_vid.count %}
@@ -91,7 +91,7 @@
                     {%- set ns_aud.count = ns_aud.count + 1 %}
                     {{- '<|mime_start|>{"id": "' + audio_id + '", "type": "audio/wav", "filename": "' + content.get('filename', "a.wav") + '"}<|mime_end|>\n' }}
                     {{- '<|audio_aux_start|>다음 중 audio_duration은 오디오 길이 정보입니다. 참고하여 답변하세요. {"audio_duration": ' + (content.get('audio_duration') | tojson if content.get('audio_duration') else '<|audio_duration|>') + '}<|audio_aux_end|>\n'}}
-                    {{- '<|discrete_audio_start|><|DISCRETE_AUDIO_PAD|><|discrete_audio_end|>'}}
+                    {{- '<|discrete_audio_start|><|DISCRETE_AUDIO_PAD|><|discrete_audio_end|>\n'}}
                     {{- '<|audio_start|><|AUDIO_PAD|><|audio_end|>'}}
                 {%- elif content['type'] == 'text' %}
                     {{- content['text'] }}

config.json

@@ -4,89 +4,28 @@
     "HyperCLOVAXOmniForCausalLM"
   ],
   "audio_config": {
-    "_name_or_path": "",
     "activation_dropout": 0.0,
     "activation_function": "gelu",
-    "add_cross_attention": false,
     "architectures": [
       "Qwen2AudioEncoder"
     ],
     "attention_dropout": 0.0,
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": null,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
     "d_model": 1280,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
     "dropout": 0.0,
-    "early_stopping": false,
     "encoder_attention_heads": 20,
     "encoder_ffn_dim": 5120,
     "encoder_layerdrop": 0.0,
     "encoder_layers": 32,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": null,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
     "init_std": 0.02,
     "initializer_range": 0.02,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "length_penalty": 1.0,
-    "max_length": 20,
     "max_source_positions": 1500,
-    "min_length": 0,
     "model_type": "qwen2_audio_encoder",
-    "no_repeat_ngram_size": 0,
-    "num_beam_groups": 1,
-    "num_beams": 1,
     "num_hidden_layers": 32,
     "num_mel_bins": 128,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": null,
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "scale_embedding": false,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": "float32",
-    "torchscript": false,
-    "typical_p": 1.0,
-    "use_bfloat16": false
+    "scale_embedding": false
   },
   "audio_end_token_id": 128257,
-  "audio_model_name_or_path": null,
   "audio_projector_type": "mlp",  
-  "audio_start_id": 128071,
   "audio_start_token_id": 128256,
   "audio_token_id": 128071,
   "auto_map": {
@@ -95,8 +34,6 @@
     "AutoModelForSequenceClassification": "modeling_hyperclovax_omni.HyperCLOVAXOmniForSequenceClassification"
   },
   "discrete_audio_config": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
     "architectures": [
       "CosyvoiceEncoder"
     ],
@@ -104,34 +41,6 @@
       "AutoConfig": "configuration_cosyvoice2.CosyVoice2Config",
       "AutoModel": "modeling_cosyvoice2.CosyVoice2Model"
     },
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": null,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "early_stopping": false,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": null,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "min_length": 0,
     "model_type": "cosyvoice2",
     "n_audio_ctx": 1500,
     "n_audio_head": 20,
@@ -139,39 +48,9 @@
     "n_audio_state": 1280,
     "n_codebook_size": 6561,
     "n_mels": 128,
-    "no_repeat_ngram_size": 0,
-    "num_beam_groups": 1,
-    "num_beams": 1,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": null,
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": "float32",
-    "torchscript": false,
-    "typical_p": 1.0,
-    "use_bfloat16": false,
     "use_sdpa": true
   },
   "discrete_audio_end_token_id": 128073,
-  "discrete_audio_model_name_or_path": null,
   "discrete_audio_start_token_id": 128072,
   "discrete_audio_token_id": 128074,
   "discrete_audio_unit_0_id": 128606,
@@ -180,9 +59,6 @@
   "discrete_image_token_id": 128069,
   "discrete_image_unit_0_id": 135168,
   "discrete_vision_config": {
-    "_name_or_path": "",
-    
-    "add_cross_attention": false,
     "architectures": [
       "TextAlignedTokenizer"
     ],
@@ -190,9 +66,6 @@
       "AutoConfig": "configuration_tatok.TATokConfig",
       "AutoModel": "modeling_tatok.TATokModel"
     },
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": null,
     "bottleneck": {
       "args": {
         "bottleneck_dim": 1536,
@@ -217,85 +90,21 @@
       "name": "bottleneck"
     },
     "bottleneck_token_num": 729,
-    "chunk_size_feed_forward": 0,
-    "ckpt_path": "google/siglip2-so400m-patch14-384",
-    "cross_attention_hidden_size": null,
     "decoder_depth": 3,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "early_stopping": false,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": null,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
     "input_size": 384,
     "input_type": "indices",
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "min_length": 0,
     "model_type": "tatok",
-    "no_repeat_ngram_size": 0,
-    "num_beam_groups": 1,
-    "num_beams": 1,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": null,
     "pool_scale": 1,
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
     "rand_scale": true,
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
     "select_layer_id": -2,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "teacher": "google/siglip2-so400m-patch14-384",
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": "float32",
-    "torchscript": false,
-    "typical_p": 1.0,
-    "use_bfloat16": false
+    "teacher": "google/siglip2-so400m-patch14-384"
   },
-  "discrete_vision_model_name_or_path": null,
-  "end_token_id": 128001,
   "eos_token_id": 128001,
-  "freeze_audio_projector": true,
-  "freeze_before_sampler": false,
-  "freeze_decoder": false,
-  "freeze_encoder": true,
-  "freeze_mm_projector": false,
-  "freeze_video_audio_compressor": false,
   "hidden_size": 4096,
   "ignore_index": -100,
   "image_end_token_id": 128058,
   "image_start_token_id": 128059,
   "image_token_id": 128062,
-  "img_start_id": 128062,
   "is_safetensor_save": true,
   "max_num_grids": -1,
   "mm_projector_type": "linear",
@@ -305,95 +114,35 @@
   "possible_resolutions": [],
   "proj_pos_emb": true,
   "proj_prenorm": false,
-  "q_former_model_name_or_path": null,
-  "skip_ve_mlp_infer": true,
   "text_config": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
     "architectures": [
       "LlamaForCausalLM"
     ],
     "attention_bias": false,
     "attention_dropout": 0.0,
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
     "bos_token_id": 128000,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "early_stopping": false,
-    "encoder_no_repeat_ngram_size": 0,
     "eos_token_id": 128001,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
     "head_dim": 128,
     "hidden_act": "silu",
     "hidden_size": 4096,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
     "initializer_range": 0.02,
     "intermediate_size": 12288,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "length_penalty": 1.0,
     "logits_scaling": 1.0,
-    "max_length": 20,
     "max_position_embeddings": 8192,
-    "min_length": 0,
     "mlp_bias": false,
     "model_type": "llama",
-    "no_repeat_ngram_size": 0,
     "num_attention_heads": 32,
-    "num_beam_groups": 1,
-    "num_beams": 1,
     "num_hidden_layers": 36,
     "num_key_value_heads": 8,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": null,
-    "prefix": null,
     "pretraining_tp": 1,
-    "problem_type": null,
-    "pruned_heads": {},
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
     "rms_norm_eps": 1e-06,
     "rope_scaling": null,
     "rope_theta": 5000000,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
     "tie_word_embeddings": false,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": "float32",
-    "torchscript": false,
-    "typical_p": 1.0,
-    "use_bfloat16": false,
     "use_cache": true,
     "vocab_size": 200704
   },
-  "text_model_name_or_path": "",
   "torch_dtype": "float32",
-  "transformers_version": "4.52.4",
   "unpad": false,
   "use_1x1_grid": false,
   "use_components": {
@@ -403,112 +152,34 @@
     "use_discrete_audio": true,
     "use_discrete_vision": true
   },
-  "use_nth_layer": -2,
+  "vision_feature_layer": -2,
   "video_audio_compressor_config": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
     "architectures": null,
     "auto_map": {
       "AutoConfig": "configuration_mambamia.MambaMiaVideoAudioCompressorConfig",
       "AutoModel": "modeling_mambamia.MambaMiaVideoAudioCompressor"
     },
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": null,
     "chunk_size": 25,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "early_stopping": false,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": null,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
     "hidden_size": 3072,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
     "input_size": 4096,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "min_length": 0,
     "model_type": "mambamia_videoaudio_compressor",
-    "no_repeat_ngram_size": 0,
-    "num_beam_groups": 1,
-    "num_beams": 1,
     "num_hidden_layers": 1,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "output_size": 4096,
-    "pad_token_id": null,
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": null,
-    "torchscript": false,
-    "typical_p": 1.0,
-    "use_bfloat16": false
+    "output_size": 4096
   },
   "video_audio_compressor_type": "mambamia",
-  "video_audio_start_id": 128070,
   "video_audio_token_id": 128070,
   "video_end_token_id": 128061,
   "video_first_last_frames_slows": null,
   "video_num_queries_fast": null,
   "video_num_queries_slow": null,
-  "video_start_id": 128063,
   "video_start_token_id": 128060,
   "video_token_id": 128063,
   "vision_config": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
     "anyres": false,
     "architectures": [
       "Qwen2_5_VisionTransformerPretrainedModel"
     ],
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": null,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
     "depth": 32,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "early_stopping": false,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": null,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
     "fullatt_block_indexes": [
       7,
       15,
@@ -517,63 +188,20 @@
     ],
     "hidden_act": "silu",
     "hidden_size": 1280,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
     "in_channels": 3,
     "in_chans": 3,
     "initializer_range": 0.02,
     "intermediate_size": 3456,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "length_penalty": 1.0,
-    "max_length": 20,
     "max_num_grids": -1,
-    "min_length": 0,
     "model_type": "qwen2_5_vl_visual",
-    "no_repeat_ngram_size": 0,
-    "num_beam_groups": 1,
-    "num_beams": 1,
     "num_heads": 16,
-    "num_return_sequences": 1,
     "out_hidden_size": 5120,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": null,
     "patch_size": 14,
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "sep_token_id": null,
     "spatial_merge_size": 2,
     "spatial_patch_size": 14,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
     "temporal_patch_size": 2,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
-    "tokenizer_class": null,
     "tokens_per_second": 2,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": "float32",
-    "torchscript": false,
-    "typical_p": 1.0,
-    "use_bfloat16": false,
     "window_size": 112
   },
-  "vision_input_chunk_size": null,
-  "vision_model_name_or_path": null
+  "vision_input_chunk_size": null
 }

configuration_cosyvoice2.py

@@ -1,107 +1,70 @@
-import transformers
-from transformers import AutoConfig, AutoModel
+# coding=utf-8
+# Copyright 2024 NAVER Cloud Corp. and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""CosyVoice2 audio encoder configuration"""
+
+from transformers import AutoConfig
 from transformers.configuration_utils import PretrainedConfig
 
-# CosyVoice (CosyvoiceEncoder) Config
+
 class CosyVoice2Config(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a CosyVoice2 audio encoder model.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        n_mels (`int`, *optional*, defaults to 128):
+            Number of mel frequency bins for the input spectrogram.
+        n_audio_ctx (`int`, *optional*, defaults to 1500):
+            Maximum audio context length.
+        n_audio_state (`int`, *optional*, defaults to 1280):
+            Dimensionality of the encoder hidden states.
+        n_audio_head (`int`, *optional*, defaults to 20):
+            Number of attention heads in the encoder.
+        n_audio_layer (`int`, *optional*, defaults to 6):
+            Number of encoder layers.
+        n_codebook_size (`int`, *optional*, defaults to 6561):
+            Size of the codebook (3^8).
+        use_sdpa (`bool`, *optional*, defaults to `True`):
+            Whether to use Scaled Dot-Product Attention.
+    """
+
     model_type = "cosyvoice2"
 
     def __init__(
         self,
-        n_mels=128,
-        n_audio_ctx=1500,
-        n_audio_state=1280,
-        n_audio_head=20,
-        n_audio_layer=6,
-        n_codebook_size=6561,  # 3**8
-        use_sdpa=True,
-        model_name_or_path=None,
+        n_mels: int = 128,
+        n_audio_ctx: int = 1500,
+        n_audio_state: int = 1280,
+        n_audio_head: int = 20,
+        n_audio_layer: int = 6,
+        n_codebook_size: int = 6561,  # 3**8
+        use_sdpa: bool = True,
         **kwargs,
     ):
-        # Extract our custom attributes from kwargs first (when coming from config.json)
-        # Priority: explicit argument (if not default) > kwargs value > default value
-        kwargs_n_mels = kwargs.pop("n_mels", 128)
-        kwargs_n_audio_ctx = kwargs.pop("n_audio_ctx", 1500)
-        kwargs_n_audio_state = kwargs.pop("n_audio_state", 1280)
-        kwargs_n_audio_head = kwargs.pop("n_audio_head", 20)
-        kwargs_n_audio_layer = kwargs.pop("n_audio_layer", 6)
-        kwargs_n_codebook_size = kwargs.pop("n_codebook_size", 6561)
-        kwargs_use_sdpa = kwargs.pop("use_sdpa", True)
-        kwargs_model_name_or_path = kwargs.pop("model_name_or_path", None)
-
-        # Apply precedence rules
-        self.n_mels = n_mels if n_mels != 128 else kwargs_n_mels
-        self.n_audio_ctx = n_audio_ctx if n_audio_ctx != 1500 else kwargs_n_audio_ctx
-        self.n_audio_state = n_audio_state if n_audio_state != 1280 else kwargs_n_audio_state
-        self.n_audio_head = n_audio_head if n_audio_head != 20 else kwargs_n_audio_head
-        self.n_audio_layer = n_audio_layer if n_audio_layer != 6 else kwargs_n_audio_layer
-        self.n_codebook_size = n_codebook_size if n_codebook_size != 6561 else kwargs_n_codebook_size
-        self.use_sdpa = use_sdpa if use_sdpa is not True else kwargs_use_sdpa
-        self.model_name_or_path = model_name_or_path if model_name_or_path is not None else kwargs_model_name_or_path
+        super().__init__(**kwargs)
+        self.n_mels = n_mels
+        self.n_audio_ctx = n_audio_ctx
+        self.n_audio_state = n_audio_state
+        self.n_audio_head = n_audio_head
+        self.n_audio_layer = n_audio_layer
+        self.n_codebook_size = n_codebook_size
+        self.use_sdpa = use_sdpa
 
-        # Default values for standard transformers config attributes
-        # These will be merged with kwargs so they appear even when not in config.json
-        default_transformers_config = {
-            "_name_or_path": None,
-            "add_cross_attention": False,
-            "architectures": ["CosyvoiceEncoder"],
-            "bad_words_ids": None,
-            "begin_suppress_tokens": None,
-            "bos_token_id": None,
-            "chunk_size_feed_forward": 0,
-            "cross_attention_hidden_size": None,
-            "decoder_start_token_id": None,
-            "diversity_penalty": 0.0,
-            "do_sample": False,
-            "early_stopping": False,
-            "encoder_no_repeat_ngram_size": 0,
-            "eos_token_id": None,
-            "exponential_decay_length_penalty": None,
-            "finetuning_task": None,
-            "forced_bos_token_id": None,
-            "forced_eos_token_id": None,
-            "id2label": {"0": "LABEL_0", "1": "LABEL_1"},
-            "is_decoder": False,
-            "is_encoder_decoder": False,
-            "label2id": {"LABEL_0": 0, "LABEL_1": 1},
-            "length_penalty": 1.0,
-            "max_length": 20,
-            "min_length": 0,
-            "no_repeat_ngram_size": 0,
-            "num_beam_groups": 1,
-            "num_beams": 1,
-            "num_return_sequences": 1,
-            "output_attentions": False,
-            "output_hidden_states": False,
-            "output_scores": False,
-            "pad_token_id": None,
-            "prefix": None,
-            "problem_type": None,
-            "pruned_heads": {},
-            "remove_invalid_values": False,
-            "repetition_penalty": 1.0,
-            "return_dict": True,
-            "return_dict_in_generate": False,
-            "sep_token_id": None,
-            "suppress_tokens": None,
-            "task_specific_params": None,
-            "temperature": 1.0,
-            "tf_legacy_loss": False,
-            "tie_encoder_decoder": False,
-            "tie_word_embeddings": True,
-            "tokenizer_class": None,
-            "top_k": 50,
-            "top_p": 1.0,
-            "torch_dtype": "float32",
-            "torchscript": False,
-            "typical_p": 1.0,
-            "use_bfloat16": False,
-        }
 
-        # Merge defaults with kwargs (kwargs values take precedence)
-        merged_kwargs = {**default_transformers_config, **kwargs}
+AutoConfig.register("cosyvoice2", CosyVoice2Config)
 
-        # Pass merged kwargs to parent (these are standard transformers config attributes)
-        super().__init__(**merged_kwargs)
-        
 __all__ = ["CosyVoice2Config"]

configuration_hyperclovax.py

@@ -1,10 +1,5 @@
 # coding=utf-8
-# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
-#
-# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
-# and OPT implementations in this library. It has been modified from its
-# original forms to accommodate minor architectural differences compared
-# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+# Copyright 2024 NAVER Cloud Corp. and the HuggingFace Inc. team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,27 +12,26 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""LLaMA model configuration"""
+"""HyperCLOVAX model configuration"""
+
+from typing import Optional
 
-import transformers
-from transformers import AutoConfig, AutoModel, AutoModelForCausalLM
+from transformers import AutoConfig
 from transformers.configuration_utils import PretrainedConfig
 
 
 class HyperCLOVAXConfig(PretrainedConfig):
     r"""
-    This is the configuration class to store the configuration of a [`LlamaModel`]. It is used to instantiate an LLaMA
-    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
-    defaults will yield a similar configuration to that of the LLaMA-7B.
+    This is the configuration class to store the configuration of a [`HyperCLOVAXModel`]. It is used to instantiate a
+    HyperCLOVAX model according to the specified arguments, defining the model architecture.
 
     Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
     documentation from [`PretrainedConfig`] for more information.
 
-
     Args:
         vocab_size (`int`, *optional*, defaults to 32000):
-            Vocabulary size of the LLaMA model. Defines the number of different tokens that can be represented by the
-            `inputs_ids` passed when calling [`LlamaModel`]
+            Vocabulary size of the HyperCLOVAX model. Defines the number of different tokens that can be represented
+            by the `inputs_ids` passed when calling [`HyperCLOVAXModel`].
         hidden_size (`int`, *optional*, defaults to 4096):
             Dimension of the hidden representations.
         intermediate_size (`int`, *optional*, defaults to 11008):
@@ -57,8 +51,7 @@ class HyperCLOVAXConfig(PretrainedConfig):
         hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
             The non-linear activation function (function or string) in the decoder.
         max_position_embeddings (`int`, *optional*, defaults to 2048):
-            The maximum sequence length that this model might ever be used with. Llama 1 supports up to 2048 tokens,
-            Llama 2 up to 4096, CodeLlama up to 16384.
+            The maximum sequence length that this model might ever be used with.
         initializer_range (`float`, *optional*, defaults to 0.02):
             The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
         rms_norm_eps (`float`, *optional*, defaults to 1e-06):
@@ -78,7 +71,7 @@ class HyperCLOVAXConfig(PretrainedConfig):
             understand more about it. This value is necessary to ensure exact reproducibility of the pretraining
             results. Please refer to [this issue](https://github.com/pytorch/pytorch/issues/76232).
         tie_word_embeddings (`bool`, *optional*, defaults to `False`):
-            Whether to tie weight embeddings
+            Whether to tie weight embeddings.
         rope_theta (`float`, *optional*, defaults to 10000.0):
             The base period of the RoPE embeddings.
         rope_scaling (`Dict`, *optional*):
@@ -125,19 +118,23 @@ class HyperCLOVAXConfig(PretrainedConfig):
         mlp_bias (`bool`, *optional*, defaults to `False`):
             Whether to use a bias in up_proj, down_proj and gate_proj layers in the MLP layers.
         head_dim (`int`, *optional*):
-            The attention head dimension. If None, it will default to hidden_size // num_heads
+            The attention head dimension. If None, it will default to hidden_size // num_heads.
+        embedding_multiplier (`float`, *optional*, defaults to 1.0):
+            Multiplier for the embedding layer output, used for muP (maximal update parametrization).
+        logits_scaling (`float`, *optional*, defaults to 1.0):
+            Scaling factor for the logits, used for muP.
+        attention_multiplier (`float`, *optional*, defaults to 1.0):
+            Multiplier for the attention scores, used for muP.
+        residual_multiplier (`float`, *optional*, defaults to 1.0):
+            Multiplier for the residual connections, used for muP.
+        use_post_norm (`bool`, *optional*, defaults to `False`):
+            Whether to apply post-normalization (dual-norm) in addition to pre-normalization.
 
     ```python
-    >>> from transformers import LlamaModel, LlamaConfig
+    >>> from configuration_hyperclovax import HyperCLOVAXConfig
 
-    >>> # Initializing a LLaMA llama-7b style configuration
-    >>> configuration = LlamaConfig()
-
-    >>> # Initializing a model from the llama-7b style configuration
-    >>> model = LlamaModel(configuration)
-
-    >>> # Accessing the model configuration
-    >>> configuration = model.config
+    >>> # Initializing a HyperCLOVAX configuration
+    >>> configuration = HyperCLOVAXConfig()
     ```"""
 
     model_type = "hyperclovax"
@@ -145,40 +142,42 @@ class HyperCLOVAXConfig(PretrainedConfig):
 
     def __init__(
         self,
-        vocab_size=32000,
-        hidden_size=4096,
-        intermediate_size=11008,
-        num_hidden_layers=32,
-        num_attention_heads=32,
-        num_key_value_heads=None,
-        hidden_act="silu",
-        max_position_embeddings=2048,
-        initializer_range=0.02,
-        rms_norm_eps=1e-6,
-        use_cache=True,
-        pad_token_id=None,
-        bos_token_id=1,
-        eos_token_id=2,
-        pretraining_tp=1,
-        tie_word_embeddings=False,
-        rope_theta=10000.0,
-        rope_scaling=None,
-        attention_bias=False,
-        attention_dropout=0.0,
-        mlp_bias=False,
-        head_dim=None,
-        embedding_multiplier=1.0,  # mup
-        logits_scaling=1.0,  # mup
-        attention_multiplier=1.0,  # mup
-        residual_multiplier=1.0,  # mup
-        use_post_norm=False,  # post-norm
-        auto_map={
-            "AutoConfig": "configuration_hyperclovax.HyperCLOVAXConfig",
-            "AutoModel": "modeling_hyperclovax.HyperCLOVAXModel",
-            "AutoModelForCausalLM": "modeling_hyperclovax.HyperCLOVAXForCausalLM",
-        },
+        vocab_size: int = 32000,
+        hidden_size: int = 4096,
+        intermediate_size: int = 11008,
+        num_hidden_layers: int = 32,
+        num_attention_heads: int = 32,
+        num_key_value_heads: Optional[int] = None,
+        hidden_act: str = "silu",
+        max_position_embeddings: int = 2048,
+        initializer_range: float = 0.02,
+        rms_norm_eps: float = 1e-6,
+        use_cache: bool = True,
+        pad_token_id: Optional[int] = None,
+        bos_token_id: int = 1,
+        eos_token_id: int = 2,
+        pretraining_tp: int = 1,
+        tie_word_embeddings: bool = False,
+        rope_theta: float = 10000.0,
+        rope_scaling: Optional[dict] = None,
+        attention_bias: bool = False,
+        attention_dropout: float = 0.0,
+        mlp_bias: bool = False,
+        head_dim: Optional[int] = None,
+        embedding_multiplier: float = 1.0,
+        logits_scaling: float = 1.0,
+        attention_multiplier: float = 1.0,
+        residual_multiplier: float = 1.0,
+        use_post_norm: bool = False,
         **kwargs,
     ):
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
         self.vocab_size = vocab_size
         self.max_position_embeddings = max_position_embeddings
         self.hidden_size = hidden_size
@@ -202,11 +201,9 @@ class HyperCLOVAXConfig(PretrainedConfig):
         self.attention_dropout = attention_dropout
         self.mlp_bias = mlp_bias
         self.head_dim = head_dim if head_dim is not None else self.hidden_size // self.num_attention_heads
-        # Validate the correctness of rotary position embeddings parameters
-        # BC: if there is a 'type' field, copy it it to 'rope_type'.
+        # BC: if there is a 'type' field, copy it to 'rope_type'.
         if self.rope_scaling is not None and "type" in self.rope_scaling:
             self.rope_scaling["rope_type"] = self.rope_scaling["type"]
-        # rope_config_validation(self)
 
         # mup
         self.embedding_multiplier = embedding_multiplier
@@ -217,13 +214,7 @@ class HyperCLOVAXConfig(PretrainedConfig):
         # post-norm (dual-norm)
         self.use_post_norm = use_post_norm
 
-        super().__init__(
-            pad_token_id=pad_token_id,
-            bos_token_id=bos_token_id,
-            eos_token_id=eos_token_id,
-            tie_word_embeddings=tie_word_embeddings,
-            auto_map=auto_map,
-            **kwargs,
-        )
-        
+
+AutoConfig.register("hyperclovax", HyperCLOVAXConfig)
+
 __all__ = ["HyperCLOVAXConfig"]

configuration_hyperclovax_omni.py

@@ -1,35 +1,134 @@
-import os
-import transformers
+# coding=utf-8
+# Copyright 2024 NAVER Cloud Corp. and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""HyperCLOVAX-Omni multimodal model configuration"""
+
+from typing import Dict, List, Optional, Union
+
 from transformers import (
-    AutoConfig, AutoModel, AutoModelForCausalLM, 
-    CLIPVisionConfig, LlamaConfig, 
-    PretrainedConfig, SiglipVisionConfig, 
-    Qwen2AudioEncoderConfig, WhisperConfig,
+    AutoConfig,
+    CLIPVisionConfig,
+    LlamaConfig,
+    PretrainedConfig,
+    Qwen2AudioEncoderConfig,
+    SiglipVisionConfig,
+    WhisperConfig,
 )
-from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLVisionConfig
+from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import Qwen2_5_VLVisionConfig
 
 from .configuration_cosyvoice2 import CosyVoice2Config
 from .configuration_hyperclovax import HyperCLOVAXConfig
 from .configuration_mambamia import MambaMiaVideoAudioCompressorConfig
 from .configuration_tatok import TATokConfig
 
+
 class HyperCLOVAXOmniConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`HyperCLOVAXOmniForCausalLM`]. It is used to
+    instantiate a HyperCLOVAX-Omni multimodal model according to the specified arguments, defining the model
+    architecture including text, vision, and audio components.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        text_config (`dict` or [`PretrainedConfig`], *optional*):
+            Configuration for the text backbone model. Accepts a `HyperCLOVAXConfig` or `LlamaConfig`.
+        vision_config (`dict` or [`PretrainedConfig`], *optional*):
+            Configuration for the continuous vision encoder (e.g., CLIP, SigLIP, Qwen2.5-VL).
+        discrete_vision_config (`dict` or [`PretrainedConfig`], *optional*):
+            Configuration for the discrete vision tokenizer (TATok).
+        audio_config (`dict` or [`PretrainedConfig`], *optional*):
+            Configuration for the continuous audio encoder (e.g., Qwen2AudioEncoder, Whisper).
+        discrete_audio_config (`dict` or [`PretrainedConfig`], *optional*):
+            Configuration for the discrete audio encoder (CosyVoice2).
+        text_model_name_or_path (`str`, *optional*):
+            Path or identifier of a pretrained text model to load config from.
+        vision_model_name_or_path (`str`, *optional*):
+            Path or identifier of a pretrained vision model to load config from.
+        discrete_vision_model_name_or_path (`str`, *optional*):
+            Path or identifier of a pretrained discrete vision model to load config from.
+        audio_model_name_or_path (`str`, *optional*):
+            Path or identifier of a pretrained audio model to load config from.
+        discrete_audio_model_name_or_path (`str`, *optional*):
+            Path or identifier of a pretrained discrete audio model to load config from.
+        mm_projector_type (`str`, *optional*, defaults to `"mlp"`):
+            Type of the multimodal projector for vision features.
+        audio_projector_type (`str`, *optional*, defaults to `"mlp"`):
+            Type of the projector for audio features.
+        video_audio_compressor_type (`str`, *optional*):
+            Type of the video-audio compressor (e.g., `"mambamia"`).
+        video_audio_compressor_config (`dict` or [`PretrainedConfig`], *optional*):
+            Configuration for the video-audio compressor module.
+        vision_feature_layer (`int`, *optional*, defaults to -2):
+            Index of the vision encoder layer to extract features from.
+        discrete_image_unit_0_id (`int`, *optional*, defaults to 135166):
+            Token id for `<|vision00000|>`, the first discrete vision unit token.
+        discrete_audio_unit_0_id (`int`, *optional*, defaults to 128604):
+            Token id for `<|audio0000|>`, the first discrete audio unit token.
+        anyres (`bool`, *optional*, defaults to `False`):
+            Whether to use any-resolution image processing.
+        unpad (`bool`, *optional*, defaults to `False`):
+            Whether to remove padding from image features.
+        max_num_grids (`int`, *optional*, defaults to -1):
+            Maximum number of grids for any-resolution processing. -1 means no limit.
+        num_queries_vis_abstractor (`int`, *optional*, defaults to -1):
+            Number of query tokens for the visual abstractor. -1 means disabled.
+        video_num_queries_fast (`int`, *optional*):
+            Number of query tokens for fast video frames.
+        video_num_queries_slow (`int`, *optional*):
+            Number of query tokens for slow video frames.
+        video_first_last_frames_slows (`int`, *optional*):
+            Number of first/last frames to process as slow frames.
+        video_max_num_frames (`int`, *optional*):
+            Maximum number of video frames to process.
+        ignore_index (`int`, *optional*, defaults to -100):
+            The index to ignore in loss computation.
+        proj_pos_emb (`bool`, *optional*, defaults to `True`):
+            Whether to use positional embeddings in the projector.
+        proj_prenorm (`bool`, *optional*, defaults to `False`):
+            Whether to apply pre-normalization in the projector.
+        use_1x1_grid (`bool`, *optional*, defaults to `False`):
+            Whether to use 1x1 grid for single-image processing.
+        possible_resolutions (`List[List[int]]`, *optional*):
+            List of possible resolutions `[height, width]` for any-resolution processing.
+
+    ```python
+    >>> from configuration_hyperclovax_omni import HyperCLOVAXOmniConfig
+
+    >>> # Initializing a HyperCLOVAX-Omni configuration
+    >>> configuration = HyperCLOVAXOmniConfig()
+    ```
+    """
     model_type = "hyperclovax_omni"
     keys_to_ignore_at_inference = ["past_key_values"]
 
-    _sub_config_attrs = [
-        "text_config",
-        "vision_config",
-        "audio_config",
-        "discrete_vision_config",
-        "discrete_audio_config",
-        "video_audio_compressor_config",
-    ]
+    sub_configs = {
+        "text_config": AutoConfig,
+        "vision_config": AutoConfig,
+        "audio_config": AutoConfig,
+        "discrete_vision_config": AutoConfig,
+        "discrete_audio_config": AutoConfig,
+        "video_audio_compressor_config": AutoConfig,
+    }
 
     @classmethod
-    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
+    def from_pretrained(cls, pretrained_model_name_or_path: str, **kwargs) -> "HyperCLOVAXOmniConfig":
         config = super().from_pretrained(pretrained_model_name_or_path, **kwargs)
-        for attr in cls._sub_config_attrs:
+        # Propagate _name_or_path to sub-configs so that AutoModel.from_config()
+        # can resolve dynamic module references (auto_map) from the checkpoint directory.
+        for attr in cls.sub_configs:
             sub_config = getattr(config, attr, None)
             if sub_config is not None and hasattr(sub_config, "_name_or_path"):
                 sub_config._name_or_path = config._name_or_path
@@ -37,56 +136,35 @@ class HyperCLOVAXOmniConfig(PretrainedConfig):
 
     def __init__(
         self,
-        text_config=None,
-        vision_config=None,
-        discrete_vision_config=None,
-        audio_config=None,
-        discrete_audio_config=None,
-        text_model_name_or_path=None,
-        vision_model_name_or_path=None,
-        discrete_vision_model_name_or_path=None,
-        audio_model_name_or_path=None,
-        discrete_audio_model_name_or_path=None,
-        q_former_model_name_or_path=None,
-        mm_projector_type="mlp",
-        audio_projector_type="mlp",
-        video_audio_compressor_type=None,
-        video_audio_compressor_config=None,
-        use_nth_layer=-2,
-        img_start_id=128062,  # <|IMAGE_PAD|> # /mnt/ddn/vuvlm/outputs/251001_spd_hcx4b_stage2_c1_withmmpt_exp3/checkpoint-1500 수동 수정해서 맞춘 상황.
-        discrete_image_start_id=128250,  # <|DISCRETE_AUDIO_PAD|>
-        discrete_image_unit_0_id=135166,  # <|vision00000|>
-        video_start_id=128063,  # <|VIDEO_PAD|>
-        video_audio_start_id=None,  # <|VIDEO_AUDIO_PAD|> - will be set dynamically
-        audio_start_id=128253,  # <|AUDIO_PAD|>
-        discrete_audio_start_id=128250,  # <|DISCRETE_AUDIO_PAD|>
-        discrete_audio_unit_0_id=128604,  # <|audio0000|>
-        freeze_encoder=False,
-        freeze_decoder=False,
-        freeze_mm_projector=False,
-        freeze_audio_projector=False,
-        freeze_video_audio_compressor=False,
-        anyres=False,
-        unpad=False,
-        max_num_grids=-1,
-        num_queries_vis_abstractor=-1,
-        video_num_queries_fast=None,
-        video_num_queries_slow=None,
-        video_first_last_frames_slows=None,
-        video_max_num_frames=None,
-        ignore_index=-100,
-        proj_pos_emb=True,
-        proj_prenorm=False,
-        use_1x1_grid=False,
-        possible_resolutions=[],
+        text_config: Optional[Union[Dict, PretrainedConfig]] = None,
+        vision_config: Optional[Union[Dict, PretrainedConfig]] = None,
+        discrete_vision_config: Optional[Union[Dict, PretrainedConfig]] = None,
+        audio_config: Optional[Union[Dict, PretrainedConfig]] = None,
+        discrete_audio_config: Optional[Union[Dict, PretrainedConfig]] = None,
+        mm_projector_type: str = "mlp",
+        audio_projector_type: str = "mlp",
+        video_audio_compressor_type: Optional[str] = None,
+        video_audio_compressor_config: Optional[Union[Dict, PretrainedConfig]] = None,
+        vision_feature_layer: int = -2,
+        discrete_image_unit_0_id: int = 135166,  # <|vision00000|>
+        discrete_audio_unit_0_id: int = 128604,  # <|audio0000|>
+        anyres: bool = False,
+        unpad: bool = False,
+        max_num_grids: int = -1,
+        num_queries_vis_abstractor: int = -1,
+        video_num_queries_fast: Optional[int] = None,
+        video_num_queries_slow: Optional[int] = None,
+        video_first_last_frames_slows: Optional[int] = None,
+        video_max_num_frames: Optional[int] = None,
+        ignore_index: int = -100,
+        proj_pos_emb: bool = True,
+        proj_prenorm: bool = False,
+        use_1x1_grid: bool = False,
+        possible_resolutions: Optional[List[List[int]]] = None,
         **kwargs,
     ):
+        super().__init__(**kwargs)
         # text_config
-        if (
-            text_config is None 
-            and text_model_name_or_path is not None
-        ):
-            text_config = AutoConfig.from_pretrained(text_model_name_or_path, trust_remote_code=True)
         if isinstance(text_config, dict):
             if text_config["model_type"] == "hyperclovax":
                 text_config = HyperCLOVAXConfig(**text_config)
@@ -94,13 +172,13 @@ class HyperCLOVAXOmniConfig(PretrainedConfig):
                 text_config = LlamaConfig(**text_config)
             else:
                 raise ValueError(f'Invalid text_config type: {text_config["model_type"]}')
-        if text_config is not None: # for deepspeed zero to allocate dynamic memory 
-            self.hidden_size = text_config.hidden_size if hasattr(text_config, "hidden_size") else text_config.n_embd
+        if text_config is not None:
+            self.hidden_size = text_config.hidden_size
+        else:
+            self.hidden_size = kwargs.get("hidden_size", 4096)
         self.text_config = text_config
-        
+
         # audio_config
-        if audio_config is None and audio_model_name_or_path is not None:
-            audio_config = AutoConfig.from_pretrained(audio_model_name_or_path)
         if isinstance(audio_config, dict):
             if audio_config["model_type"] == "qwen2_audio_encoder":
                 audio_config = Qwen2AudioEncoderConfig(**audio_config)
@@ -109,26 +187,13 @@ class HyperCLOVAXOmniConfig(PretrainedConfig):
             else:
                 raise ValueError(f'Invalid audio_config type: {audio_config["model_type"]}')
         self.audio_config = audio_config
-        
+
         # discrete_audio_config
-        if discrete_audio_config is None and discrete_audio_model_name_or_path is not None:
-            discrete_audio_config = {
-                "model_type": "cosyvoice2",
-                "model_name_or_path": discrete_audio_model_name_or_path,
-            }
-        if (
-            isinstance(CosyVoice2Config, type)
-            and isinstance(discrete_audio_config, dict)
-        ):
+        if isinstance(discrete_audio_config, dict):
             discrete_audio_config = CosyVoice2Config.from_dict(discrete_audio_config)
         self.discrete_audio_config = discrete_audio_config
-        
+
         # vision_config
-        if vision_config is None and vision_model_name_or_path is not None:
-            vision_config = AutoConfig.from_pretrained(
-                vision_model_name_or_path, 
-                trust_remote_code=True,
-            )
         if isinstance(vision_config, dict):
             if vision_config["model_type"] == "clip_vision_model":
                 vision_config = CLIPVisionConfig(**vision_config)
@@ -139,16 +204,8 @@ class HyperCLOVAXOmniConfig(PretrainedConfig):
             else:
                 raise ValueError(f'Invalid vision_config type: {vision_config["model_type"]}')
         self.vision_config = vision_config
-            
+
         # discrete_vision_config
-        if (
-            discrete_vision_config is None 
-            and discrete_vision_model_name_or_path is not None
-        ):
-            discrete_vision_config = {
-                "model_type": "tatok",
-                "model_name_or_path": discrete_vision_model_name_or_path,
-            }
         if isinstance(discrete_vision_config, dict):
             discrete_vision_config = TATokConfig.from_dict(discrete_vision_config)
         self.discrete_vision_config = discrete_vision_config
@@ -159,21 +216,10 @@ class HyperCLOVAXOmniConfig(PretrainedConfig):
         self.video_audio_compressor_config = video_audio_compressor_config
 
         # add VLM configs
-        self.text_model_name_or_path = text_model_name_or_path
-        self.vision_model_name_or_path = vision_model_name_or_path
-        self.discrete_vision_model_name_or_path = discrete_vision_model_name_or_path
-        self.audio_model_name_or_path = audio_model_name_or_path
-        self.discrete_audio_model_name_or_path = discrete_audio_model_name_or_path
-        self.q_former_model_name_or_path = q_former_model_name_or_path
         self.mm_projector_type = mm_projector_type
         self.audio_projector_type = audio_projector_type
         self.video_audio_compressor_type = video_audio_compressor_type
-        self.use_nth_layer = use_nth_layer
-        self.freeze_encoder = freeze_encoder
-        self.freeze_decoder = freeze_decoder
-        self.freeze_mm_projector = freeze_mm_projector
-        self.freeze_audio_projector = freeze_audio_projector
-        self.freeze_video_audio_compressor = freeze_video_audio_compressor
+        self.vision_feature_layer = vision_feature_layer
         self.anyres = anyres
         self.unpad = unpad
         self.max_num_grids = max_num_grids
@@ -183,39 +229,20 @@ class HyperCLOVAXOmniConfig(PretrainedConfig):
         self.video_first_last_frames_slows = video_first_last_frames_slows
         self.video_max_num_frames = video_max_num_frames
 
-        self.img_start_id = img_start_id
-        self.image_token_id = img_start_id
-
-        self.discrete_image_start_id = discrete_image_start_id
-        self.discrete_image_token_id = discrete_image_start_id
         self.discrete_image_unit_0_id = discrete_image_unit_0_id
-
-        self.video_start_id = video_start_id
-        self.video_token_id = video_start_id
-
-        self.video_audio_start_id = video_audio_start_id
-        self.video_audio_token_id = video_audio_start_id
-
-        self.audio_start_id = audio_start_id
-        self.audio_token_id = audio_start_id
-
-        self.discrete_audio_start_id = discrete_audio_start_id
-        self.discrete_audio_token_id = discrete_audio_start_id
         self.discrete_audio_unit_0_id = discrete_audio_unit_0_id
 
         self.ignore_index = ignore_index
         self.proj_pos_emb = proj_pos_emb
         self.proj_prenorm = proj_prenorm
         self.use_1x1_grid = use_1x1_grid
-        self.possible_resolutions = possible_resolutions
-        
-        super().__init__(**kwargs)
-        
+        self.possible_resolutions = possible_resolutions if possible_resolutions is not None else []
+
         # needed for HCXVisionForSequenceClassification
-        if self.text_config is not None: 
+        if self.text_config is not None:
             self.pad_token_id = self.text_config.pad_token_id
-            
-from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLVisionConfig
 
 
-__all__ = ["CosyVoice2Config", "HyperCLOVAXConfig", "HyperCLOVAXOmniConfig", "MambaMiaVideoAudioCompressorConfig", "TATokConfig", "Qwen2_5_VLVisionConfig", ]
+AutoConfig.register("hyperclovax_omni", HyperCLOVAXOmniConfig)
+
+__all__ = ["HyperCLOVAXOmniConfig"]

configuration_mambamia.py

@@ -1,19 +1,41 @@
-import transformers
-from transformers import AutoConfig, AutoModel
+# coding=utf-8
+# Copyright 2024 NAVER Cloud Corp. and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""MambaMia video-audio compressor configuration"""
+
+from transformers import AutoConfig
 from transformers.configuration_utils import PretrainedConfig
 
 
-# MambaMia VideoAudio Compressor Config
 class MambaMiaVideoAudioCompressorConfig(PretrainedConfig):
-    """
-    Configuration for MambaMiaVideoAudioCompressor.
+    r"""
+    This is the configuration class to store the configuration of a MambaMia video-audio compressor.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
 
     Args:
-        input_size: Input embedding dimension (e.g., 1280 for Whisper)
-        output_size: Output embedding dimension (e.g., 2048 for LLM)
-        chunk_size: Number of tokens per chunk (default: 25, i.e., 1 second at 25Hz)
-        num_hidden_layers: Number of MambaMia2 layers (default: 1)
-        hidden_size: Internal hidden size (default: 3072, must be divisible by 24)
+        input_size (`int`, *optional*, defaults to 1280):
+            Input embedding dimension (e.g., 1280 for Whisper encoder output).
+        output_size (`int`, *optional*, defaults to 2048):
+            Output embedding dimension (e.g., 2048 for LLM hidden size).
+        chunk_size (`int`, *optional*, defaults to 25):
+            Number of tokens per chunk (i.e., 1 second at 25Hz).
+        num_hidden_layers (`int`, *optional*, defaults to 1):
+            Number of MambaMia2 layers.
+        hidden_size (`int`, *optional*, defaults to 3072):
+            Internal hidden size. Must be divisible by 24.
     """
 
     model_type = "mambamia_videoaudio_compressor"
@@ -33,6 +55,8 @@ class MambaMiaVideoAudioCompressorConfig(PretrainedConfig):
         self.chunk_size = chunk_size
         self.num_hidden_layers = num_hidden_layers
         self.hidden_size = hidden_size
-        
-        
+
+
+AutoConfig.register("mambamia_videoaudio_compressor", MambaMiaVideoAudioCompressorConfig)
+
 __all__ = ["MambaMiaVideoAudioCompressorConfig"]

configuration_tatok.py

@@ -1,13 +1,57 @@
-import transformers
-from transformers import AutoConfig, AutoModel
+# coding=utf-8
+# Copyright 2024 NAVER Cloud Corp. and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""TATok (Text-Aligned Tokenizer) configuration"""
+
+import copy
+from typing import Optional
+
+from transformers import AutoConfig
 from transformers.configuration_utils import PretrainedConfig
 
 
-# TATok (TextAlignedTokenizer) Config
 class TATokConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a TATok (Text-Aligned Tokenizer) model.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        bottleneck (`dict`, *optional*):
+            Configuration dict for the bottleneck layer. If not provided, uses `TATokConfig._default_bottleneck`.
+        bottleneck_token_num (`int`, *optional*, defaults to 729):
+            Number of bottleneck tokens.
+        input_size (`int`, *optional*, defaults to 384):
+            Input image size.
+        teacher (`str`, *optional*, defaults to `"google/siglip2-so400m-patch14-384"`):
+            Name or path of the teacher model.
+        input_type (`str`, *optional*, defaults to `"indices"`):
+            Type of input representation.
+        pool_scale (`int`, *optional*, defaults to 1):
+            Pooling scale factor.
+        decoder_depth (`int`, *optional*, defaults to 3):
+            Number of decoder layers.
+        select_layer_id (`int`, *optional*, defaults to -2):
+            Index of the teacher layer to select features from.
+        rand_scale (`bool`, *optional*, defaults to `True`):
+            Whether to use random scaling during training.
+    """
+
     model_type = "tatok"
 
-    default_tatok_bottleneck = {
+    _default_bottleneck = {
         "name": "bottleneck",
         "args": {
             "bottleneck_dim": 1536,
@@ -33,125 +77,29 @@ class TATokConfig(PretrainedConfig):
 
     def __init__(
         self,
-        bottleneck=None,
-        bottleneck_token_num=729,
-        input_size=384,
-        teacher="google/siglip2-so400m-patch14-384",
-        input_type="indices",
-        pool_scale=1,
-        decoder_depth=3,
-        select_layer_id=-2,
-        model_name_or_path=None,
+        bottleneck: Optional[dict] = None,
+        bottleneck_token_num: int = 729,
+        input_size: int = 384,
+        teacher: str = "google/siglip2-so400m-patch14-384",
+        input_type: str = "indices",
+        pool_scale: int = 1,
+        decoder_depth: int = 3,
+        select_layer_id: int = -2,
+        rand_scale: bool = True,
         **kwargs,
     ):
-        # Extract our custom attributes from kwargs first
-        # This handles the case when config.json contains these values
-        # Priority: explicit argument (if not default) > kwargs value > default value
-        kwargs_bottleneck = kwargs.pop("bottleneck", None)
-        kwargs_bottleneck_token_num = kwargs.pop("bottleneck_token_num", 729)
-        kwargs_input_size = kwargs.pop("input_size", 384)
-        kwargs_teacher = kwargs.pop("teacher", "google/siglip2-so400m-patch14-384")
-        kwargs_input_type = kwargs.pop("input_type", "indices")
-        kwargs_pool_scale = kwargs.pop("pool_scale", 1)
-        kwargs_decoder_depth = kwargs.pop("decoder_depth", 3)
-        kwargs_select_layer_id = kwargs.pop("select_layer_id", -2)
-        kwargs_model_name_or_path = kwargs.pop("model_name_or_path", None)
-
-        # Also handle ckpt_path which might be in kwargs from config.json
-        ckpt_path = kwargs.pop("ckpt_path", None)
-        if ckpt_path is not None:
-            kwargs_teacher = ckpt_path
-            # Store ckpt_path to include in merged_kwargs later
-            stored_ckpt_path = ckpt_path
-        else:
-            stored_ckpt_path = None
-
-        # Use explicit arguments if they differ from defaults, otherwise use kwargs values.
-        # For bottleneck, if everything is None, fall back to the full default hierarchical config.
-        if bottleneck is not None:
-            self.bottleneck = bottleneck
-        elif kwargs_bottleneck is not None:
-            self.bottleneck = kwargs_bottleneck
-        else:
-            self.bottleneck = self.default_tatok_bottleneck
-
-        self.bottleneck_token_num = bottleneck_token_num if bottleneck_token_num != 729 else kwargs_bottleneck_token_num
-        self.input_size = input_size if input_size != 384 else kwargs_input_size
-        self.teacher = teacher if teacher != "google/siglip2-so400m-patch14-384" else kwargs_teacher
-        self.input_type = input_type if input_type != "indices" else kwargs_input_type
-        self.pool_scale = pool_scale if pool_scale != 1 else kwargs_pool_scale
-        self.decoder_depth = decoder_depth if decoder_depth != 3 else kwargs_decoder_depth
-        self.select_layer_id = select_layer_id if select_layer_id != -2 else kwargs_select_layer_id
-        self.model_name_or_path = model_name_or_path if model_name_or_path is not None else kwargs_model_name_or_path
-
-        # Set default values for standard transformers config attributes
-        # These will be merged with kwargs so they appear even when not in config.json
-        default_transformers_config = {
-            "_name_or_path": None,
-            "add_cross_attention": False,
-            "architectures": ["TextAlignedTokenizer"],
-            "bad_words_ids": None,
-            "begin_suppress_tokens": None,
-            "bos_token_id": None,
-            "chunk_size_feed_forward": 0,
-            "cross_attention_hidden_size": None,
-            "decoder_start_token_id": None,
-            "diversity_penalty": 0.0,
-            "do_sample": False,
-            "early_stopping": False,
-            "encoder_no_repeat_ngram_size": 0,
-            "eos_token_id": None,
-            "exponential_decay_length_penalty": None,
-            "finetuning_task": None,
-            "forced_bos_token_id": None,
-            "forced_eos_token_id": None,
-            "id2label": {"0": "LABEL_0", "1": "LABEL_1"},
-            "is_decoder": False,
-            "is_encoder_decoder": False,
-            "label2id": {"LABEL_0": 0, "LABEL_1": 1},
-            "length_penalty": 1.0,
-            "max_length": 20,
-            "min_length": 0,
-            "no_repeat_ngram_size": 0,
-            "num_beam_groups": 1,
-            "num_beams": 1,
-            "num_return_sequences": 1,
-            "output_attentions": False,
-            "output_hidden_states": False,
-            "output_scores": False,
-            "pad_token_id": None,
-            "prefix": None,
-            "problem_type": None,
-            "pruned_heads": {},
-            "rand_scale": True,
-            "remove_invalid_values": False,
-            "repetition_penalty": 1.0,
-            "return_dict": True,
-            "return_dict_in_generate": False,
-            "sep_token_id": None,
-            "suppress_tokens": None,
-            "task_specific_params": None,
-            "temperature": 1.0,
-            "tf_legacy_loss": False,
-            "tie_encoder_decoder": False,
-            "tie_word_embeddings": True,
-            "tokenizer_class": None,
-            "top_k": 50,
-            "top_p": 1.0,
-            "torch_dtype": "float32",
-            "torchscript": False,
-            "typical_p": 1.0,
-            "use_bfloat16": False,
-        }
+        super().__init__(**kwargs)
+        self.bottleneck = bottleneck if bottleneck is not None else copy.deepcopy(self._default_bottleneck)
+        self.bottleneck_token_num = bottleneck_token_num
+        self.input_size = input_size
+        self.teacher = teacher
+        self.input_type = input_type
+        self.pool_scale = pool_scale
+        self.decoder_depth = decoder_depth
+        self.select_layer_id = select_layer_id
+        self.rand_scale = rand_scale
 
-        # Merge defaults with kwargs (kwargs values take precedence)
-        merged_kwargs = {**default_transformers_config, **kwargs}
 
-        # Add ckpt_path back if it was in the original kwargs
-        if stored_ckpt_path is not None:
-            merged_kwargs["ckpt_path"] = stored_ckpt_path
+AutoConfig.register("tatok", TATokConfig)
 
-        # Pass merged kwargs to parent (these are standard transformers config attributes)
-        super().__init__(**merged_kwargs)
-        
 __all__ = ["TATokConfig"]

image_processing_hyperclovax_omni.py

@@ -1,19 +1,19 @@
 """
-HyperCLOVAXOmni Image Processor (Fast)
+HyperCLOVAX Omni Image Processor (Fast)
 
-Qwen VL 스타일의 동적 해상도 이미지 처리를 구현합니다:
-- Smart resize: 이미지를 min_pixels와 max_pixels 범위 내로 조정
-- Vision token calculation: merge_size를 사용한 토큰 축소
-- Discrete image processing: 별도의 discrete vision token 처리
+Implements dynamic resolution image processing:
+- Smart resize: adjusts image to fit within min_pixels and max_pixels
+- Vision token calculation: token reduction using merge_size
+- Discrete image processing: separate processing for discrete vision tokens
 
-BaseImageProcessorFast 기반으로 torchvision resize를 사용하여
-Track A (Qwen2VLImageProcessorFast)와 bit-perfect 일치를 달성합니다.
+Based on BaseImageProcessorFast with torchvision resize.
 """
 
-import os
 import math
+import os
+from typing import List, Optional, Tuple, Union
+
 import torch
-from typing import Dict, List, Optional, Union, Tuple
 from torchvision.transforms.v2 import functional as F
 from transformers.image_processing_utils import BatchFeature
 from transformers.image_processing_utils_fast import (
@@ -32,29 +32,31 @@ from transformers.image_utils import (
 from transformers.processing_utils import ImagesKwargs, Unpack
 from transformers.utils import TensorType, logging
 
-
 logger = logging.get_logger(__name__)
 
 
 def smart_resize(
-    height: int, width: int, factor: int = 28, min_pixels: int = 56 * 56, max_pixels: int = 14 * 14 * 4 * 1280
-):
-    """
-    Qwen VL 스타일 스마트 리사이즈
-
-    이미지 크기를 다음 조건을 만족하도록 조정:
-    1. 양쪽 차원이 factor로 나누어떨어짐
-    2. 총 픽셀 수가 min_pixels와 max_pixels 사이
+    height: int,
+    width: int,
+    factor: int = 28,
+    min_pixels: int = 56 * 56,
+    max_pixels: int = 14 * 14 * 4 * 1280,
+) -> Tuple[int, int]:
+    """Smart resize for dynamic resolution.
+
+    Adjusts image dimensions to satisfy:
+    1. Both dimensions are divisible by factor.
+    2. Total pixel count is between min_pixels and max_pixels.
 
     Args:
-        height: 원본 이미지 높이
-        width: 원본 이미지 너비
-        factor: 라운딩 단위 (기본: 28 = patch_size * merge_size)
-        min_pixels: 최소 픽셀 수 (기본: 3136)
-        max_pixels: 최대 픽셀 수 (기본: 1003520)
+        height: Original image height.
+        width: Original image width.
+        factor: Rounding unit (default: 28 = patch_size * merge_size).
+        min_pixels: Minimum pixel count (default: 3136).
+        max_pixels: Maximum pixel count (default: 1003520).
 
     Returns:
-        (new_height, new_width) 튜플
+        Tuple of (new_height, new_width).
     """
     if max(height, width) / min(height, width) > 200:
         raise ValueError(
@@ -73,18 +75,22 @@ def smart_resize(
     return h_bar, w_bar
 
 
-def calculate_qwen_num_patches(height: int, width: int, patch_size: int = 14, merge_size: int = 2):
-    """
-    Calculate number of vision tokens using Qwen VL method
+def calculate_num_patches(
+    height: int,
+    width: int,
+    patch_size: int = 14,
+    merge_size: int = 2,
+) -> int:
+    """Calculate the number of vision tokens.
 
     Args:
-        height: Image height (should be divisible by patch_size * merge_size)
-        width: Image width (should be divisible by patch_size * merge_size)
-        patch_size: ViT patch size (default: 14)
-        merge_size: Merge size for token reduction (default: 2)
+        height: Image height (should be divisible by patch_size * merge_size).
+        width: Image width (should be divisible by patch_size * merge_size).
+        patch_size: ViT patch size (default: 14).
+        merge_size: Merge size for token reduction (default: 2).
 
     Returns:
-        Number of vision tokens
+        Number of vision tokens.
     """
     grid_h = height // patch_size
     grid_w = width // patch_size
@@ -98,14 +104,6 @@ class HyperCLOVAXOmniFastImageProcessorKwargs(DefaultFastImageProcessorKwargs, t
     patch_size: Optional[int]
     temporal_patch_size: Optional[int]
     merge_size: Optional[int]
-
-
-class HyperCLOVAXOmniImagesKwargs(ImagesKwargs, total=False):
-    min_pixels: Optional[int]
-    max_pixels: Optional[int]
-    patch_size: Optional[int]
-    temporal_patch_size: Optional[int]
-    merge_size: Optional[int]
     # Token parameters
     image_token: Optional[str]
     image_start_token: Optional[str]
@@ -123,19 +121,15 @@ class HyperCLOVAXOmniImagesKwargs(ImagesKwargs, total=False):
 
 
 class HyperCLOVAXOmniImageProcessor(BaseImageProcessorFast):
-    """
-    Fast image processor for HyperCLOVAXOmni that follows Qwen VL's image processing logic.
+    """Fast image processor for HyperCLOVAX Omni.
 
-    Uses torchvision-based resize (BaseImageProcessorFast) for bit-perfect alignment
-    with Qwen2VLImageProcessorFast.
-
-    This processor implements Qwen VL's dynamic resolution approach:
-    1. Smart resize: adjusts image size to be within min_pixels and max_pixels
-    2. Vision token calculation: uses merge_size for token reduction
-    3. Discrete image processing: separate processing for discrete vision tokens
+    Uses torchvision-based resize for dynamic resolution processing:
+    1. Smart resize: adjusts image size to be within min_pixels and max_pixels.
+    2. Vision token calculation: uses merge_size for token reduction.
+    3. Discrete image processing: separate processing for discrete vision tokens.
     """
 
-    # Class-level defaults (Qwen2VLImageProcessorFast pattern)
+    # Class-level defaults
     resample = PILImageResampling.BICUBIC
     image_mean = OPENAI_CLIP_MEAN
     image_std = OPENAI_CLIP_STD
@@ -154,7 +148,7 @@ class HyperCLOVAXOmniImageProcessor(BaseImageProcessorFast):
     valid_kwargs = HyperCLOVAXOmniFastImageProcessorKwargs
 
     def __init__(self, **kwargs):
-        # Handle size ↔ min_pixels/max_pixels (Qwen2VL pattern)
+        # Handle size <-> min_pixels/max_pixels
         size = kwargs.pop("size", None)
         min_pixels = kwargs.pop("min_pixels", None)
         max_pixels = kwargs.pop("max_pixels", None)
@@ -192,7 +186,7 @@ class HyperCLOVAXOmniImageProcessor(BaseImageProcessorFast):
         if self.max_pixels is None:
             self.max_pixels = self.size["longest_edge"]
 
-        # Build ratio → token mapping from discrete_image_ratios
+        # Build ratio -> token mapping from discrete_image_ratios
         ratios = self.discrete_image_ratios if self.discrete_image_ratios is not None else []
         self.discrete_image_ratio_tokens = {
             f"{r[0]}:{r[1]}": f"<|vision_ratio_{r[0]}:{r[1]}|>"
@@ -206,7 +200,7 @@ class HyperCLOVAXOmniImageProcessor(BaseImageProcessorFast):
         max_pixels: Optional[int] = None,
         **kwargs,
     ) -> dict:
-        """size ↔ min_pixels/max_pixels 상호 연동"""
+        """Synchronize size <-> min_pixels/max_pixels."""
         if min_pixels is not None and max_pixels is not None:
             size = {"shortest_edge": min_pixels, "longest_edge": max_pixels}
         elif size is not None:
@@ -219,12 +213,41 @@ class HyperCLOVAXOmniImageProcessor(BaseImageProcessorFast):
 
         return super()._further_process_kwargs(size=size, min_pixels=min_pixels, max_pixels=max_pixels, **kwargs)
 
+    def _find_best_ratio_token(
+        self,
+        original_size: List[int],
+        discrete_image_ratios: Optional[List[List[int]]] = None,
+    ) -> Tuple[int, int]:
+        """Find the best ratio token based on the original image aspect ratio.
+
+        Args:
+            original_size: Original [height, width] of the image.
+            discrete_image_ratios: List of [h, w] ratio pairs. Defaults to self.discrete_image_ratios.
+
+        Returns:
+            Best matching (h_ratio, w_ratio) tuple.
+        """
+        discrete_image_ratios = discrete_image_ratios if discrete_image_ratios is not None else self.discrete_image_ratios
+
+        if not discrete_image_ratios:
+            return (1, 1)
+
+        h, w = original_size
+        if h == 0 or w == 0:
+            return (1, 1)
+
+        ratios = [i / j for i, j in discrete_image_ratios]
+        diffs = [abs(w / h - r) for r in ratios]
+        best_size_idx = diffs.index(min(diffs))
+
+        return discrete_image_ratios[best_size_idx]
+
     def _preprocess_continuous_image(
         self,
         images: list,
         do_resize: bool,
         size: SizeDict,
-        interpolation: Optional["F.InterpolationMode"],
+        interpolation: Optional[F.InterpolationMode],
         do_rescale: bool,
         rescale_factor: float,
         do_normalize: bool,
@@ -235,29 +258,30 @@ class HyperCLOVAXOmniImageProcessor(BaseImageProcessorFast):
         merge_size: int,
         disable_grouping: Optional[bool],
     ) -> dict:
-        """이미지에 대한 continuous image 전처리를 수행합니다.
+        """Preprocess images for continuous vision features.
 
-        Qwen2VL 패턴에 따라 smart resize → rescale+normalize → patchify를 수행합니다.
+        Performs smart resize -> rescale+normalize -> patchify.
 
         Args:
-            images: 전처리할 이미지 텐서 리스트.
-            do_resize: 리사이즈 수행 여부.
-            size: min_pixels/max_pixels를 포함하는 SizeDict.
-            interpolation: 보간 방법.
-            do_rescale: rescale 수행 여부.
-            rescale_factor: rescale 계수.
-            do_normalize: normalize 수행 여부.
-            image_mean: 정규화 평균.
-            image_std: 정규화 표준편차.
-            patch_size: ViT 패치 크기.
-            temporal_patch_size: 시간 축 패치 크기.
-            merge_size: 토큰 축소를 위한 merge 크기.
-            disable_grouping: 그룹화 비활성화 여부.
+            images: List of image tensors to preprocess.
+            do_resize: Whether to perform resizing.
+            size: SizeDict containing min_pixels/max_pixels.
+            interpolation: Interpolation method.
+            do_rescale: Whether to perform rescaling.
+            rescale_factor: Rescale factor.
+            do_normalize: Whether to perform normalization.
+            image_mean: Normalization mean.
+            image_std: Normalization standard deviation.
+            patch_size: ViT patch size.
+            temporal_patch_size: Temporal patch size.
+            merge_size: Token reduction merge size.
+            disable_grouping: Whether to disable image grouping.
 
         Returns:
-            dict with:
-                - "pixel_values": (N, num_patches, patch_dim) 텐서.
-                - "image_grid_thw": (N, 3) 텐서.
+            Dictionary with:
+                - "pixel_values": Tensor of shape (N, num_patches, patch_dim).
+                - "image_grid_thw": Tensor of shape (N, 3).
+                - "num_image_tokens": Tensor of shape (N,) with per-image token counts.
         """
         # 1. Group & smart resize
         grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping)
@@ -279,7 +303,7 @@ class HyperCLOVAXOmniImageProcessor(BaseImageProcessorFast):
             resized_images_grouped[shape] = stacked_images
         resized_images = reorder_images(resized_images_grouped, grouped_images_index)
 
-        # 2. Group again → fused rescale+normalize → patchify
+        # 2. Group again -> fused rescale+normalize -> patchify
         grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping)
         processed_images_grouped = {}
         processed_grids = {}
@@ -303,7 +327,7 @@ class HyperCLOVAXOmniImageProcessor(BaseImageProcessorFast):
             grid_t = grid_t // temporal_patch_size
             grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
 
-            # Patchify: reshape → permute → flatten
+            # Patchify: reshape -> permute -> flatten
             patches = patches.view(
                 batch_size,
                 grid_t, temporal_patch_size,
@@ -338,22 +362,23 @@ class HyperCLOVAXOmniImageProcessor(BaseImageProcessorFast):
         self,
         images: list,
         original_sizes: List[Tuple[int, int]],
-        interpolation: Optional["F.InterpolationMode"],
+        interpolation: Optional[F.InterpolationMode],
     ) -> dict:
-        """이미지에 대한 discrete image 전처리를 수행합니다.
+        """Preprocess images for discrete vision tokens.
 
-        각 이미지를 고정 크기(discrete_image_size)로 리사이즈하고,
-        원본 비율에 가장 가까운 ratio token을 찾습니다.
+        Resizes each image to a fixed size (discrete_image_size) and finds
+        the closest aspect ratio token.
 
         Args:
-            images: 전처리할 이미지 텐서 리스트.
-            original_sizes: 각 이미지의 원본 (height, width) 튜플 리스트.
-            interpolation: 보간 방법.
+            images: List of image tensors to preprocess.
+            original_sizes: List of (height, width) tuples for each image.
+            interpolation: Interpolation method.
 
         Returns:
-            dict with:
-                - "discrete_pixel_values": (N, C, discrete_image_size, discrete_image_size) 텐서.
-                - "discrete_image_ratios": (N, 2) 텐서.
+            Dictionary with:
+                - "discrete_pixel_values": Tensor of shape (N, C, discrete_image_size, discrete_image_size).
+                - "discrete_image_ratios": Tensor of shape (N, 2).
+                - "num_discrete_image_tokens": Tensor of shape (N,) with per-image discrete token counts.
         """
         discrete_pixel_values_list = []
         discrete_image_ratios_list = []
@@ -393,7 +418,7 @@ class HyperCLOVAXOmniImageProcessor(BaseImageProcessorFast):
         images: list,
         do_resize: bool,
         size: SizeDict,
-        interpolation: Optional["F.InterpolationMode"],
+        interpolation: Optional[F.InterpolationMode],
         do_rescale: bool,
         rescale_factor: float,
         do_normalize: bool,
@@ -405,12 +430,36 @@ class HyperCLOVAXOmniImageProcessor(BaseImageProcessorFast):
         disable_grouping: Optional[bool],
         return_tensors: Optional[Union[str, TensorType]],
         **kwargs,
-    ):
+    ) -> BatchFeature:
+        """Main preprocessing entry point called by BaseImageProcessorFast.
+
+        Performs continuous image processing and optionally discrete image processing.
+
+        Args:
+            images: List of image tensors.
+            do_resize: Whether to perform resizing.
+            size: SizeDict containing min_pixels/max_pixels.
+            interpolation: Interpolation method.
+            do_rescale: Whether to perform rescaling.
+            rescale_factor: Rescale factor.
+            do_normalize: Whether to perform normalization.
+            image_mean: Normalization mean.
+            image_std: Normalization standard deviation.
+            patch_size: ViT patch size.
+            temporal_patch_size: Temporal patch size.
+            merge_size: Token reduction merge size.
+            disable_grouping: Whether to disable image grouping.
+            return_tensors: Desired tensor type for outputs.
+
+        Returns:
+            BatchFeature containing pixel_values, image_grid_thw, and optionally
+            discrete processing results.
+        """
         # Record original sizes for discrete processing before any transforms
         if self.use_discrete_image_token:
             original_sizes = [(img.shape[-2], img.shape[-1]) for img in images]
 
-        # --- Continuous processing ---
+        # Continuous processing
         continuous_result = self._preprocess_continuous_image(
             images,
             do_resize=do_resize,
@@ -428,7 +477,7 @@ class HyperCLOVAXOmniImageProcessor(BaseImageProcessorFast):
         )
         data = continuous_result
 
-        # --- Discrete processing ---
+        # Discrete processing
         if self.use_discrete_image_token:
             discrete_result = self._preprocess_discrete_image(
                 images,
@@ -439,39 +488,35 @@ class HyperCLOVAXOmniImageProcessor(BaseImageProcessorFast):
 
         return BatchFeature(data=data, tensor_type=return_tensors)
 
-    def _find_best_ratio_token(
-        self,
-        original_size: List[int],
-        discrete_image_ratios: Optional[List[List[int]]] = None,
-    ):
-        """Find the best ratio token based on original_size"""
-        discrete_image_ratios = discrete_image_ratios if discrete_image_ratios is not None else self.discrete_image_ratios
-
-        if not discrete_image_ratios:
-            return (1, 1)
-
-        h, w = original_size
-        if h == 0 or w == 0:
-            return (1, 1)
-
-        ratios = [i / j for i, j in discrete_image_ratios]
-        diffs = [abs(w / h - r) for r in ratios]
-        best_size_idx = diffs.index(min(diffs))
-
-        return discrete_image_ratios[best_size_idx]
-
     def get_num_image_tokens(
         self,
         image_width: Optional[int] = None,
         image_height: Optional[int] = None,
-        pixel_values: Optional["torch.Tensor"] = None,
+        pixel_values: Optional[torch.Tensor] = None,
         include_boundary_tokens: bool = False,
         min_pixels: Optional[int] = None,
         max_pixels: Optional[int] = None,
         patch_size: Optional[int] = None,
         merge_size: Optional[int] = None,
+        return_tuple: Optional[bool] = None,
     ) -> int:
-        """이미지 입력에 대한 토큰 수를 계산합니다."""
+        """Compute the number of image tokens for the given input.
+
+        Args:
+            image_width: Image width (used when pixel_values is None).
+            image_height: Image height (used when pixel_values is None).
+            pixel_values: Pre-computed pixel values tensor.
+            include_boundary_tokens: Whether to include start/end boundary tokens.
+            min_pixels: Minimum pixel count. Defaults to self.min_pixels.
+            max_pixels: Maximum pixel count. Defaults to self.max_pixels.
+            patch_size: ViT patch size. Defaults to self.patch_size.
+            merge_size: Token reduction merge size. Defaults to self.merge_size.
+            return_tuple: If True, return (continuous, discrete) tuple.
+                Otherwise return the sum.
+
+        Returns:
+            Token count as int, or (continuous, discrete) tuple if return_tuple is True.
+        """
         patch_size = patch_size if patch_size is not None else self.patch_size
         merge_size = merge_size if merge_size is not None else self.merge_size
         min_pixels = min_pixels if min_pixels is not None else self.min_pixels
@@ -483,7 +528,7 @@ class HyperCLOVAXOmniImageProcessor(BaseImageProcessorFast):
             resized_height, resized_width = smart_resize(
                 image_height, image_width, factor=factor, min_pixels=min_pixels, max_pixels=max_pixels
             )
-            num_patches = calculate_qwen_num_patches(resized_height, resized_width, patch_size, merge_size)
+            num_patches = calculate_num_patches(resized_height, resized_width, patch_size, merge_size)
             num_continuous_tokens = num_patches // (merge_size ** 2)
         elif len(pixel_values.shape) == 2:
             num_continuous_tokens = pixel_values.shape[0] // (merge_size ** 2)
@@ -492,23 +537,31 @@ class HyperCLOVAXOmniImageProcessor(BaseImageProcessorFast):
                 pv.shape[0] // (merge_size ** 2) for pv in pixel_values
             )
         if include_boundary_tokens:
-            num_continuous_tokens += 2  # <|image_start_token, <|image_end_token
+            num_continuous_tokens += 2
 
         if self.use_discrete_image_token:
             discrete_token_size = self.discrete_token_size
-            num_discrete_tokens = discrete_token_size ** 2 # 729
-            # num_discrete_tokens += discrete_token_size  # vision_eol
-            # num_discrete_tokens += 1  # vision_eof
+            num_discrete_tokens = discrete_token_size ** 2
             if include_boundary_tokens:
-                num_discrete_tokens += 2  # <|discrete_image_start_token|>, <|discrete_image_end_token|>
+                num_discrete_tokens += 2
 
-        return (num_continuous_tokens, num_discrete_tokens)
+        if return_tuple:
+            return (num_continuous_tokens, num_discrete_tokens)
+        else:
+            return num_continuous_tokens + num_discrete_tokens
 
     def save_pretrained(
         self,
         save_directory: Union[str, os.PathLike],
         *args,
         **kwargs,
-    ):
+    ) -> None:
+        """Save the processor to a directory.
+
+        Registers for auto class before saving.
+
+        Args:
+            save_directory: Directory path to save the processor.
+        """
         self.register_for_auto_class()
         super().save_pretrained(save_directory, *args, **kwargs)

modeling_hyperclovax_omni.py

@@ -154,10 +154,6 @@ class HyperCLOVAXOmniModel(HyperCLOVAXOmniPreTrainedModel):
             else:
                 self.config.possible_resolutions = config.possible_resolutions
 
-        if without_llm:
-            # if vision_config.vision_module_type not in ["officialllava", "cream2"]:
-            # service에서, "vision_model_name_or_path" 의 경로가 vuclip_name2save_path 에 있는 default경로가 아니라, custom한 경로를 따라가야함.
-            vision_config.vison_pretrained_name_or_path = config.vision_model_name_or_path
         with no_init_weights():
             if self.is_qwen_visual and is_ampere_or_newer():
                 vision_config._attn_implementation = "flash_attention_2"
@@ -302,7 +298,7 @@ class HyperCLOVAXOmniModel(HyperCLOVAXOmniPreTrainedModel):
                 hidden_features=input_hidden_size,  # TODO: llava 처럼 hidden_size 를 input_hidden_size 가 아니라 LLM embedding size 로 바꿔주기
                 out_features=text_config.hidden_size,
             )
-        self.use_nth_layer = config.use_nth_layer
+        self.vision_feature_layer = config.vision_feature_layer
         self.model_parallel = False
         self.device_map = None
         self.vision_model_use_no_grad = None
@@ -346,7 +342,7 @@ class HyperCLOVAXOmniModel(HyperCLOVAXOmniPreTrainedModel):
     ) -> Union[Tuple, CausalLMOutputWithPast]:
         """
         :param input_ids: torch.int64 : torch.size([batchsize, variable)]) : SystemPrompt with Question text token indices for tokenizer.
-         In positions where images are inputted, the value is replaced by config.img_start_id, which is a vocabulary index used to indicate the start of image data.
+         In positions where images are inputted, the value is replaced by config.image_token_id, which is a vocabulary index used to indicate the start of image data.
         :param pixel_values: List of List of 4D tensor (torch.float32)
          Each outer list corresponds to a batch and contains inner lists, each holding tensors for images in a sample. The structure accounts for samples with multiple images.
         :param past_key_values: None
@@ -511,7 +507,7 @@ class HyperCLOVAXOmniModel(HyperCLOVAXOmniPreTrainedModel):
     ):
         """
         :param input_ids: torch.int64 : torch.size([batchsize, variable)]) : SystemPrompt with Question text token indices for tokenizer.
-         In positions where images are inputted, the value is replaced by config.img_start_id, which is a vocabulary index used to indicate the start of image data.
+         In positions where images are inputted, the value is replaced by config.image_token_id, which is a vocabulary index used to indicate the start of image data.
          In cases where a sample contains no images, a single dummy image is included.
         :param pixel_values: List of List of 4D tensor (torch.float32)
          Each outer list corresponds to a batch and contains inner lists, each holding tensors for images in a sample. The structure accounts for samples with multiple images.
@@ -537,7 +533,7 @@ class HyperCLOVAXOmniModel(HyperCLOVAXOmniPreTrainedModel):
         else:
             if self.is_qwen_visual:
                 inputs_embeds = self.get_input_embeddings()(input_ids)
-                context_vision_model = torch.no_grad() if self.config.freeze_encoder else contextlib.nullcontext()
+                context_vision_model = contextlib.nullcontext()
 
                 if pixel_values is not None:
                     with context_vision_model:
@@ -545,11 +541,11 @@ class HyperCLOVAXOmniModel(HyperCLOVAXOmniPreTrainedModel):
                     image_features = self.mm_projector(image_features)
 
                     if img_start_ids_list is None:
-                        image_cnts = (input_ids == self.config.img_start_id).sum(dim=1).tolist()
+                        image_cnts = (input_ids == self.config.image_token_id).sum(dim=1).tolist()
                     else:
                         image_cnts = [len(img_start_ids) for img_start_ids in img_start_ids_list]
 
-                    mask = input_ids.eq(self.config.img_start_id)
+                    mask = input_ids.eq(self.config.image_token_id)
                     positions = mask.nonzero(as_tuple=False)
 
                     batch_idx = positions[:, 0]
@@ -564,8 +560,8 @@ class HyperCLOVAXOmniModel(HyperCLOVAXOmniPreTrainedModel):
                         video_features = self.vision_model(pixel_values_videos, grid_thw=video_grid_thw)
                     video_features = self.mm_projector(video_features)
 
-                    video_cnts = (input_ids == self.config.video_start_id).sum(dim=1).tolist()
-                    mask = input_ids.eq(self.config.video_start_id)
+                    video_cnts = (input_ids == self.config.video_token_id).sum(dim=1).tolist()
+                    mask = input_ids.eq(self.config.video_token_id)
                     positions = mask.nonzero(as_tuple=False)
 
                     batch_idx = positions[:, 0]
@@ -674,8 +670,8 @@ class HyperCLOVAXOmniModel(HyperCLOVAXOmniPreTrainedModel):
                     if current_chunk_size == 0:
                         chunk = dummy
 
-                    # vision 모델에 chunk를 통과시킴 (use_nth_layer에 따라 처리)
-                    if self.use_nth_layer == -1:
+                    # vision 모델에 chunk를 통과시킴 (vision_feature_layer에 따라 처리)
+                    if self.vision_feature_layer == -1:
                         # 마지막 레이어의 후처리인 post_layernorm을 Identity로 대체
                         self.vision_model.vision_model.post_layernorm = nn.Identity()
                         with context_vision_model:
@@ -684,7 +680,7 @@ class HyperCLOVAXOmniModel(HyperCLOVAXOmniPreTrainedModel):
                     else:
                         with context_vision_model:
                             outs = self.vision_model(chunk, output_hidden_states=True)
-                        outs = outs.hidden_states[self.use_nth_layer][:, visual_token_idx:]
+                        outs = outs.hidden_states[self.vision_feature_layer][:, visual_token_idx:]
                     if self.vision_model_use_no_grad:
                         outs = outs.detach().requires_grad_(True)
                     if not is_adaptive_anyres:
@@ -728,7 +724,7 @@ class HyperCLOVAXOmniModel(HyperCLOVAXOmniPreTrainedModel):
                     image_forward_outs = torch.cat(image_forward_outs, dim=0).to(image_forward_outs[0].dtype)
 
                 if img_start_ids_list is None:
-                    image_cnts = (input_ids == self.config.img_start_id).sum(dim=1).tolist()
+                    image_cnts = (input_ids == self.config.image_token_id).sum(dim=1).tolist()
                 else:
                     image_cnts = [len(img_start_ids) for img_start_ids in img_start_ids_list]
 
@@ -781,13 +777,13 @@ class HyperCLOVAXOmniModel(HyperCLOVAXOmniPreTrainedModel):
                 image_feature_dtype = image_features[0][0].dtype
 
                 if img_start_ids_list is None:
-                    image_cnts = (input_ids == self.config.img_start_id).sum(dim=1).tolist()
+                    image_cnts = (input_ids == self.config.image_token_id).sum(dim=1).tolist()
                 else:
                     image_cnts = [len(img_start_ids) for img_start_ids in img_start_ids_list]
 
                 if non_vision_query_lengths is None:
                     non_vision_query_lengths = self.determine_non_vision_query_lengths(
-                        input_ids, self.config.text_config.pad_token_id, self.config.img_start_id
+                        input_ids, self.config.text_config.pad_token_id, self.config.image_token_id
                     )
 
                 if vision_query_lengths is None:
@@ -832,7 +828,7 @@ class HyperCLOVAXOmniModel(HyperCLOVAXOmniPreTrainedModel):
                         ]  # batch_idx sample 의 첫번째 이미지 (dummy 이미지)
                     else:
                         if img_start_ids_list is None:
-                            img_start_ids = (sample == self.config.img_start_id).nonzero()
+                            img_start_ids = (sample == self.config.image_token_id).nonzero()
                         else:
                             img_start_ids = img_start_ids_list[batch_idx]
                         assert len(img_start_ids) == image_cnts[batch_idx] == len(image_features[batch_idx])
@@ -1171,7 +1167,7 @@ class HyperCLOVAXOmniForCausalLM(HyperCLOVAXOmniPreTrainedModel, GenerationMixin
     ) -> Union[Tuple, CausalLMOutputWithPast]:
         """
         :param input_ids: torch.int64 : torch.size([batchsize, variable)]) : SystemPrompt with Question text token indices for tokenizer.
-         In positions where images are inputted, the value is replaced by config.img_start_id, which is a vocabulary index used to indicate the start of image data.
+         In positions where images are inputted, the value is replaced by config.image_token_id, which is a vocabulary index used to indicate the start of image data.
         :param pixel_values: List of List of 4D tensor (torch.float32)
          Each outer list corresponds to a batch and contains inner lists, each holding tensors for images in a sample. The structure accounts for samples with multiple images.
         :param past_key_values: None
@@ -1272,7 +1268,7 @@ class HyperCLOVAXOmniForCausalLM(HyperCLOVAXOmniPreTrainedModel, GenerationMixin
     ):
         """
         :param input_ids: torch.int64 : torch.size([batchsize, variable)]) : SystemPrompt with Question text token indices for tokenizer.
-         In positions where images are inputted, the value is replaced by config.img_start_id, which is a vocabulary index used to indicate the start of image data.
+         In positions where images are inputted, the value is replaced by config.image_token_id, which is a vocabulary index used to indicate the start of image data.
          In cases where a sample contains no images, a single dummy image is included.
         :param pixel_values: List of List of 4D tensor (torch.float32)
          Each outer list corresponds to a batch and contains inner lists, each holding tensors for images in a sample. The structure accounts for samples with multiple images.

preprocessor_config.json

@@ -78,7 +78,7 @@
   "video_do_resize": true,
   "video_end_token": "<|video_end|>",
   "video_max_frames": 120,
-  "video_max_pixels": 602112,
+  "video_max_pixels": 12845056,
   "video_mean": [
     0.48145466,
     0.4578275,

processing_hyperclovax_omni.py

@@ -833,24 +833,7 @@ class HyperCLOVAXOmniProcessor(ProcessorMixin):
             and self.image_processor.use_discrete_image_token
         ):
             data.update(discrete_image_inputs)
-            
-        # _tensorable_data, _untensorable_data = dict(), dict()
-        # for _k, _v in data.items():
-        #     if _v is None:
-        #         continue
-        #     if isinstance(_v, list) and any(x is None for x in _v):
-        #         continue
-        #     if _k in [
-        #         "discrete_image_ratios",
-        #         "num_audio_tokens",
-        #         "num_discrete_audio_tokens",
-        #     ]:
-        #         _untensorable_data[_k] = _v
-        #     else:
-        #         _tensorable_data[_k] = _v
-
-        # model_inputs = BatchFeature(data=_tensorable_data, tensor_type=return_tensors)
-        # model_inputs.update(_untensorable_data)
+
         model_inputs = BatchFeature(data=data, tensor_type=return_tensors)
         return model_inputs
 
@@ -860,143 +843,159 @@ class HyperCLOVAXOmniProcessor(ProcessorMixin):
     ):
         audio_placeholder = ""
         if self.audio_processor.use_discrete_audio_token:
-            audio_placeholder += f'{self.audio_processor.discrete_audio_start_token}{self.audio_processor.discrete_audio_token}{self.audio_processor.discrete_audio_end_token}'
+            audio_placeholder += f'{self.audio_processor.discrete_audio_start_token}{self.audio_processor.discrete_audio_token}{self.audio_processor.discrete_audio_end_token}\n'
         audio_placeholder += f'{self.audio_processor.audio_start_token}{self.audio_processor.audio_token}{self.audio_processor.audio_end_token}'
         if tokenize:
             audio_placeholder = self.tokenizer.encode(audio_placeholder)
         return audio_placeholder
 
-    def get_image_placeholder(
-        self,
-        tokenize: bool = False,
-    ):
-        image_placeholder = ""
-        if self.image_processor.use_discrete_audio_token:
-            image_placeholder += f'{self.image_processor.discrete_image_start_token}{self.image_processor.discrete_image_token}{self.image_processor.discrete_image_end_token}'
-        image_placeholder += f'{self.image_processor.image_start_token}{self.image_processor.image_token}{self.image_processor.image_end_token}'
-        if tokenize:
-            image_placeholder = self.tokenizer.encode(image_placeholder)
-        return image_placeholder
-        
-    def get_video_placeholder(
-        self,
-        tokenize: bool = False,
-    ):
-        video_placeholder = f'{self.video_processor.video_start_token}{self.video_processor.video_token}{self.video_processor.video_end_token}'
-        if tokenize:
-            video_placeholder = self.tokenizer.encode(video_placeholder)
-        return video_placeholder
-
     def get_audio_token_replacement(
         self,
         num_audio_tokens: int,
+        num_discrete_audio_tokens: Optional[int] = None,
         include_boundary_tokens: Optional[bool] = False,
         tokenize: Optional[bool] = False,
+        return_tuple: Optional[bool] = None,
     ):
+        conitnuous_replacement, discrete_replacement = "", ""
+        
         if (
             isinstance(num_audio_tokens, (list, tuple))
             or (isinstance(num_audio_tokens, torch.Tensor) and num_audio_tokens.dim() >= 1)
         ):
             num_audio_tokens = num_audio_tokens[0]
-            
-        replacement = self.audio_processor.audio_token * num_audio_tokens
+        conitnuous_replacement = self.audio_processor.audio_token * num_audio_tokens
         if include_boundary_tokens:
-            replacement = f"{self.audio_processor.audio_start_token}{replacement}{self.audio_processor.audio_end_token}"
-        if tokenize:
-            replacement = self.tokenizer.encode(replacement)
-        return replacement
-    
-    def get_discrete_audio_token_replacement(
+            conitnuous_replacement = f"{self.audio_processor.audio_start_token}{conitnuous_replacement}{self.audio_processor.audio_end_token}"
+        
+        if self.audio_processor.use_discrete_audio_token:
+            if (
+                isinstance(num_discrete_audio_tokens, (list, tuple))
+                or (isinstance(num_discrete_audio_tokens, torch.Tensor) and num_discrete_audio_tokens.dim() >= 1)
+            ):
+                num_discrete_audio_tokens = num_discrete_audio_tokens[0]    
+            discrete_replacement = self.audio_processor.discrete_audio_token * num_discrete_audio_tokens
+            if include_boundary_tokens:
+                discrete_replacement = f"{self.audio_processor.discrete_audio_start_token}{discrete_replacement}{self.audio_processor.discrete_audio_end_token}"
+            discrete_replacement = f'{discrete_replacement}\n'
+        
+        if return_tuple:
+            if tokenize:
+                conitnuous_replacement = self.tokenizer.encode(conitnuous_replacement)
+                discrete_replacement = self.tokenizer.encode(discrete_replacement)
+            return (conitnuous_replacement, discrete_replacement)
+        else:
+            replacement = f'{discrete_replacement}{conitnuous_replacement}'
+            if tokenize:
+                replacement = self.tokenizer.encode(replacement)
+            return replacement
+
+    def get_image_placeholder(
         self,
-        num_discrete_audio_tokens: int,
-        include_boundary_tokens: Optional[bool] = False,
-        tokenize: Optional[bool] = False,
+        tokenize: bool = False,
     ):
-        if (
-            isinstance(num_discrete_audio_tokens, (list, tuple))
-            or (isinstance(num_discrete_audio_tokens, torch.Tensor) and num_discrete_audio_tokens.dim() >= 1)
-        ):
-            num_discrete_audio_tokens = num_discrete_audio_tokens[0]
-            
-        replacement = self.audio_processor.discrete_audio_token * num_discrete_audio_tokens
-        if include_boundary_tokens:
-            replacement = f"{self.audio_processor.discrete_audio_start_token}{replacement}{self.audio_processor.discrete_audio_end_token}"
+        image_placeholder = ""
+        if self.image_processor.use_discrete_audio_token:
+            image_placeholder += f'{self.image_processor.discrete_image_start_token}{self.image_processor.discrete_image_token}{self.image_processor.discrete_image_end_token}\n'
+        image_placeholder += f'{self.image_processor.image_start_token}{self.image_processor.image_token}{self.image_processor.image_end_token}'
         if tokenize:
-            replacement = self.tokenizer.encode(replacement)
-        return replacement
+            image_placeholder = self.tokenizer.encode(image_placeholder)
+        return image_placeholder
 
     def get_image_token_replacement(
         self,
         num_image_tokens: int,
+        discrete_image_ratio: Optional[List[int]] = None,
         include_boundary_tokens: Optional[bool] = False,
         tokenize: Optional[bool] = False,
+        return_tuple: Optional[bool] = None,
     ):
+        conitnuous_replacement, discrete_replacement = "", ""
+        
         if (
             isinstance(num_image_tokens, (list, tuple))
             or (isinstance(num_image_tokens, torch.Tensor) and num_image_tokens.dim() >= 1)
         ):
             num_image_tokens = num_image_tokens[0]
-            
         discrete_token_size = self.image_processor.discrete_token_size
-        replacement = self.image_processor.image_token * num_image_tokens
+        continuous_replacement = self.image_processor.image_token * num_image_tokens
         if include_boundary_tokens:
-            replacement = f"{self.image_processor.image_start_token}{replacement}{self.image_processor.image_end_token}"
-        if tokenize:
-            replacement = self.tokenizer.encode(replacement)
-        return replacement
-    
-    def get_discrete_image_token_replacement(
+            continuous_replacement = f"{self.image_processor.image_start_token}{continuous_replacement}{self.image_processor.image_end_token}"
+        
+        if self.image_processor.use_discrete_image_token:
+            if (
+                discrete_image_ratio
+                and len(discrete_image_ratio) == 1
+                and (
+                    isinstance(discrete_image_ratio, (list, tuple)) 
+                    or (isinstance(discrete_image_ratio, torch.Tensor) and discrete_image_ratio.dim() >= 2)
+                )
+            ): # [[16, 9]], or torch.Tensor([[16, 9]])
+                discrete_image_ratio = discrete_image_ratio[0]
+            row_str = self.image_processor.discrete_image_token * self.image_processor.discrete_token_size
+            # row_str += self.image_processor.vision_eol_token
+            discrete_replacement = row_str * self.image_processor.discrete_token_size
+            if discrete_image_ratio:
+                if isinstance(discrete_image_ratio, (list, tuple)):
+                    ratio_key = f"{int(discrete_image_ratio[0])}:{int(discrete_image_ratio[1])}"
+                elif isinstance(discrete_image_ratio, torch.Tensor):
+                    ratio_key = f"{discrete_image_ratio[0].item()}:{discrete_image_ratio[1].item()}"
+                discrete_image_ratio_token = self.image_processor.discrete_image_ratio_tokens[ratio_key]
+                discrete_replacement = f"{discrete_image_ratio_token}{discrete_replacement}"
+            # discrete_replacement = f"{discrete_replacement}{self.image_processor.vision_eof_token}"
+            if include_boundary_tokens:
+                discrete_replacement = f"{self.image_processor.discrete_image_start_token}{discrete_replacement}{self.image_processor.discrete_image_end_token}"
+            discrete_replacement = f'{discrete_replacement}\n'
+
+        if return_tuple:
+            if tokenize:
+                conitnuous_replacement = self.tokenizer.encode(conitnuous_replacement)
+                discrete_replacement = self.tokenizer.encode(discrete_replacement)
+            return (conitnuous_replacement, discrete_replacement)
+        else:
+            replacement = f'{discrete_replacement}{conitnuous_replacement}'
+            if tokenize:
+                replacement = self.tokenizer.encode(replacement)
+            return replacement
+
+    def get_video_placeholder(
         self,
-        discrete_image_ratio: Optional[List[int]] = None,
-        include_boundary_tokens: Optional[bool] = False,
-        tokenize: Optional[bool] = False,
+        tokenize: bool = False,
     ):
-        if (
-            discrete_image_ratio
-            and len(discrete_image_ratio) == 1
-            and (
-                isinstance(discrete_image_ratio, (list, tuple)) 
-                or (isinstance(discrete_image_ratio, torch.Tensor) and discrete_image_ratio.dim() >= 2)
-            )
-        ): # [[16, 9]], or torch.Tensor([[16, 9]])
-            discrete_image_ratio = discrete_image_ratio[0]
-        
-        row_str = self.image_processor.discrete_image_token * self.image_processor.discrete_token_size
-        # row_str += self.image_processor.vision_eol_token
-        replacement = row_str * self.image_processor.discrete_token_size
-        if discrete_image_ratio:
-            if isinstance(discrete_image_ratio, (list, tuple)):
-                ratio_key = f"{int(discrete_image_ratio[0])}:{int(discrete_image_ratio[1])}"
-            elif isinstance(discrete_image_ratio, torch.Tensor):
-                ratio_key = f"{discrete_image_ratio[0].item()}:{discrete_image_ratio[1].item()}"
-            discrete_image_ratio_token = self.image_processor.discrete_image_ratio_tokens[ratio_key]
-            replacement = f"{discrete_image_ratio_token}{replacement}"
-        # replacement = f"{replacement}{self.image_processor.vision_eof_token}"
-        if include_boundary_tokens:
-            replacement = f"{self.image_processor.discrete_image_start_token}{replacement}{self.image_processor.discrete_image_end_token}"
+        video_placeholder = f'{self.video_processor.video_start_token}{self.video_processor.video_token}{self.video_processor.video_end_token}'
         if tokenize:
-            replacement = self.tokenizer.encode(replacement)
-        return replacement
+            video_placeholder = self.tokenizer.encode(video_placeholder)
+        return video_placeholder
 
     def get_video_token_replacement(
         self,
         num_video_tokens: int,
         include_boundary_tokens: Optional[bool] = False,
         tokenize: Optional[bool] = False,
+        return_tuple: Optional[bool] = None,
     ):
+        conitnuous_replacement, discrete_replacement = "", ""
+        
         if (
             isinstance(num_video_tokens, (list, tuple))
             or (isinstance(num_video_tokens, torch.Tensor) and num_video_tokens.dim() >= 1)
         ):
             num_video_tokens = num_video_tokens[0]
-            
         merge_length = self.video_processor.video_merge_size**2
-        replacement = self.video_processor.video_token * int(num_video_tokens)
+        conitnuous_replacement = self.video_processor.video_token * int(num_video_tokens)
         if include_boundary_tokens:
-            replacement = f"{self.video_processor.video_start_token}{replacement}{self.video_processor.video_end_token}"
-        if tokenize:
-            replacement = self.tokenizer.encode(replacement)
-        return replacement
+            conitnuous_replacement = f"{self.video_processor.video_start_token}{conitnuous_replacement}{self.video_processor.video_end_token}"
+            
+        if return_tuple:
+            if tokenize:
+                conitnuous_replacement = self.tokenizer.encode(conitnuous_replacement)
+                discrete_replacement = self.tokenizer.encode(discrete_replacement)
+            return (conitnuous_replacement, discrete_replacement)
+        else:
+            replacement = f'{discrete_replacement}{conitnuous_replacement}'
+            if tokenize:
+                replacement = self.tokenizer.encode(replacement)
+            return replacement
 
     def _interleave_video_audio_tokens(
         self,

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:666f303c324b9b2e2e8f13950cd44a18896a6fc1a70aae70583a77663d0ebe31
-size 23621510
+oid sha256:25be15af870991ef21ee479f5ca5f72ab4160e51e6e6a6131b65505ec32b63c4
+size 23621753

tokenizer_config.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e6c389295385be8eaa613b30db4a988065eac05eb1351b427dd385a6bc4cea5
-size 13220218
+oid sha256:7fa862c7a52c316ae1c3e81d74f36688d970eb56a367957473bf185eff64a259
+size 13220251

video_processing_hyperclovax_omni.py

@@ -1,46 +1,61 @@
 """
-HyperCLOVAXOmni Video Processor
+HyperCLOVAX Omni Video Processor
 
-Qwen2VL 스타일의 동적 해상도 비디오 처리를 구현합니다:
-- Smart resize: 비디오 프레임을 min_pixels와 max_pixels 범위 내로 조정
-- Temporal patch: temporal_patch_size 단위로 프레임 그룹핑
-- Patch flattening: merge_size를 사용한 토큰 축소
+Implements dynamic resolution video processing:
+- Smart resize: adjusts video frames to fit within min_pixels and max_pixels
+- Temporal patch: frame grouping by temporal_patch_size
+- Patch flattening: token reduction using merge_size
 
-BaseVideoProcessor 기반으로 torchvision resize를 사용하여
-Track A (Qwen2VLVideoProcessor)와 bit-perfect 일치를 달성합니다.
-
-입력 준비(numpy→torch), kwargs 해석, resample→interpolation 변환 등은
-BaseVideoProcessor.preprocess()와 동일한 흐름을 따릅니다.
-_preprocess_continuous_video는 Qwen2VLVideoProcessor._preprocess와
-동일한 시그니처·로직을 사용합니다.
+Based on BaseVideoProcessor with torchvision resize.
 """
 
 import math
+from typing import Dict, List, Optional, Tuple, Union
+
 import numpy as np
 import torch
-from typing import Dict, List, Optional, Union
-
+from torchvision.transforms.v2 import functional as F
 from transformers.image_processing_utils import BatchFeature
+from transformers.image_processing_utils_fast import pil_torch_interpolation_mapping
 from transformers.image_transforms import to_channel_dimension_format
 from transformers.image_utils import (
     OPENAI_CLIP_MEAN,
     OPENAI_CLIP_STD,
-    PILImageResampling,
     ChannelDimension,
-    get_image_size,
+    PILImageResampling,
     SizeDict,
+    get_image_size,
 )
+from transformers.processing_utils import VideosKwargs
 from transformers.utils import TensorType, logging
 from transformers.video_processing_utils import BaseVideoProcessor
 from transformers.video_utils import group_videos_by_shape, reorder_videos
-from transformers.processing_utils import VideosKwargs
 
 logger = logging.get_logger(__name__)
 
 
 def smart_resize(
-    height: int, width: int, factor: int = 28, min_pixels: int = 56 * 56, max_pixels: int = 14 * 14 * 4 * 1280
-):
+    height: int,
+    width: int,
+    factor: int = 28,
+    min_pixels: int = 56 * 56,
+    max_pixels: int = 14 * 14 * 4 * 1280,
+) -> Tuple[int, int]:
+    """Smart resize for dynamic resolution.
+
+    Adjusts dimensions so that both sides are divisible by factor
+    and total pixel count is between min_pixels and max_pixels.
+
+    Args:
+        height: Original height.
+        width: Original width.
+        factor: Rounding unit (default: 28 = patch_size * merge_size).
+        min_pixels: Minimum pixel count.
+        max_pixels: Maximum pixel count.
+
+    Returns:
+        Tuple of (new_height, new_width).
+    """
     if max(height, width) / min(height, width) > 200:
         raise ValueError(
             f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}"
@@ -82,11 +97,10 @@ class HyperCLOVAXOmniVideosKwargs(VideosKwargs, total=False):
 
 
 class HyperCLOVAXOmniVideoProcessor(BaseVideoProcessor):
-    """
-    Video processor for HyperCLOVAXOmni that follows Qwen2VL's video processing logic.
+    """Video processor for HyperCLOVAX Omni.
 
-    BaseVideoProcessor 기반으로 torchvision resize와 rescale_and_normalize를 사용하여
-    Qwen2VLVideoProcessor와 bit-perfect 일치를 달성합니다.
+    Uses torchvision resize and rescale_and_normalize for
+    dynamic resolution video processing.
     """
 
     model_input_names = ["pixel_values_videos", "video_grid_thw"]
@@ -117,7 +131,7 @@ class HyperCLOVAXOmniVideoProcessor(BaseVideoProcessor):
         vision_eof_token: str = "<|vision_eof|>",
         **kwargs,
     ):
-        # Qwen2VLVideoProcessor와 동일하게 size dict를 구성하여 super().__init__에 전달
+        # Construct size dict (shortest_edge/longest_edge)
         size = {"shortest_edge": video_min_pixels, "longest_edge": video_max_pixels}
 
         super().__init__(
@@ -149,10 +163,10 @@ class HyperCLOVAXOmniVideoProcessor(BaseVideoProcessor):
 
     def _preprocess_continuous_video(
         self,
-        videos: List["torch.Tensor"],
+        videos: List[torch.Tensor],
         do_resize: bool,
         size: SizeDict,
-        interpolation: "torch.nn.functional.InterpolationMode",
+        interpolation: F.InterpolationMode,
         do_rescale: bool,
         rescale_factor: float,
         do_normalize: bool,
@@ -161,33 +175,31 @@ class HyperCLOVAXOmniVideoProcessor(BaseVideoProcessor):
         patch_size: int,
         temporal_patch_size: int,
         merge_size: int,
-    ) -> Dict:
-        """단일 비디오에 대한 continuous vision 전처리를 수행합니다.
+    ) -> dict:
+        """Preprocess a single video for continuous vision features.
 
-        Qwen2VLVideoProcessor._preprocess와 동일한 시그니처·로직:
-        group_videos_by_shape → self.resize (torchvision) → self.rescale_and_normalize → patchify
+        Performs group_videos_by_shape -> resize (torchvision) -> rescale_and_normalize -> patchify.
 
         Args:
-            videos: channel-first torch 텐서 리스트. 각 원소는 (num_frames, C, H, W).
-            do_resize: 리사이즈 수행 여부.
-            size: SizeDict with shortest_edge / longest_edge (smart_resize의 min/max pixels).
+            videos: List of channel-first torch tensors, each of shape (num_frames, C, H, W).
+            do_resize: Whether to perform resizing.
+            size: SizeDict with shortest_edge/longest_edge (smart_resize min/max pixels).
             interpolation: torchvision InterpolationMode.
-            do_rescale: rescale 수행 여부.
-            rescale_factor: rescale 배율.
-            do_normalize: 정규화 수행 여부.
-            image_mean: 정규화 평균 (tuple).
-            image_std: 정규화 표준편차 (tuple).
-            patch_size: ViT patch 크기.
-            temporal_patch_size: 시간 축 patch 크기.
-            merge_size: token merge 크기.
+            do_rescale: Whether to perform rescaling.
+            rescale_factor: Rescale factor.
+            do_normalize: Whether to perform normalization.
+            image_mean: Normalization mean (tuple).
+            image_std: Normalization standard deviation (tuple).
+            patch_size: ViT patch size.
+            temporal_patch_size: Temporal patch size.
+            merge_size: Token merge size.
 
         Returns:
-            dict with:
-                - "pixel_values_videos": (grid_t * grid_h * grid_w, feat_dim) 텐서.
-                - "video_grid_thw": [grid_t, grid_h, grid_w] 리스트.
-                - "num_video_tokens": continuous 토큰 수 (int).
+            Dictionary with:
+                - "pixel_values_videos": Tensor of shape (grid_t * grid_h * grid_w, feat_dim).
+                - "video_grid_thw": List of [grid_t, grid_h, grid_w].
+                - "num_video_tokens": Number of continuous tokens (int).
         """
-        # --- Qwen2VLVideoProcessor._preprocess와 동일한 흐름 ---
         grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
 
         resized_videos_grouped = {}
@@ -258,11 +270,15 @@ class HyperCLOVAXOmniVideoProcessor(BaseVideoProcessor):
             "num_video_tokens": num_video_tokens,
         }
 
-    def _preprocess_discrete_video(
-        self,
-        video: "torch.Tensor",
-    ) -> Dict:
-        """단일 비디오에 대한 discrete vision 전처리 (현재 미구현)."""
+    def _preprocess_discrete_video(self, video: torch.Tensor) -> dict:
+        """Preprocess a single video for discrete vision tokens (not yet implemented).
+
+        Args:
+            video: Video tensor.
+
+        Returns:
+            Empty dictionary.
+        """
         return {}
 
     def preprocess(
@@ -271,28 +287,29 @@ class HyperCLOVAXOmniVideoProcessor(BaseVideoProcessor):
         return_tensors: Optional[Union[str, TensorType]] = None,
         **kwargs,
     ) -> BatchFeature:
-        """비디오 배치에 대한 전처리를 수행합니다.
+        """Preprocess a batch of videos.
 
-        BaseVideoProcessor.preprocess()와 동일한 흐름:
-        1. numpy → channel-first torch 변환 (_prepare_input_videos 패턴)
-        2. kwargs 해석 및 resample→interpolation, mean/std→tuple 변환 (_further_process_kwargs 패턴)
-        3. _preprocess_continuous_video 호출 (Qwen2VL._preprocess 시그니처)
+        Follows the same flow as BaseVideoProcessor.preprocess():
+        1. numpy (NHWC) -> channel-first torch (NCHW) conversion
+        2. kwargs resolution and resample -> interpolation, mean/std -> tuple conversion
+        3. _preprocess_continuous_video call per video
 
         Args:
-            videos: 비디오 입력. 다음 형태 중 하나:
-                - np.ndarray: 단일 비디오 (num_frames, H, W, C).
-                - List[np.ndarray]: 배치된 비디오들, 각 원소는 4D.
+            videos: Video input. Either:
+                - np.ndarray: Single video of shape (num_frames, H, W, C).
+                - List[np.ndarray]: Batch of videos, each 4D.
+            return_tensors: Desired tensor type for outputs.
 
         Returns:
             BatchFeature with:
-                - pixel_values_videos: (total_patches, feat_dim) 텐서.
-                - video_grid_thw: (num_videos, 3) 텐서.
-                - num_video_tokens: (num_videos,) 텐서.
+                - pixel_values_videos: Tensor of shape (total_patches, feat_dim).
+                - video_grid_thw: Tensor of shape (num_videos, 3).
+                - num_video_tokens: Tensor of shape (num_videos,).
         """
         if isinstance(videos, np.ndarray) and videos.ndim == 4:
             videos = [videos]
 
-        # --- 1. kwargs 해석: self 속성에서 기본값 채우기 ---
+        # 1. Resolve kwargs from self attributes
         do_resize = kwargs.pop("do_resize", kwargs.pop("video_do_resize", None))
         if do_resize is None:
             do_resize = self.do_resize
@@ -337,7 +354,7 @@ class HyperCLOVAXOmniVideoProcessor(BaseVideoProcessor):
         if merge_size is None:
             merge_size = self.video_merge_size
 
-        # size dict: Qwen2VLVideoProcessor와 동일하게 shortest_edge/longest_edge 사용
+        # Size dict (shortest_edge/longest_edge)
         min_pixels = kwargs.pop("min_pixels", kwargs.pop("video_min_pixels", None))
         if min_pixels is None:
             min_pixels = self.size["shortest_edge"]
@@ -348,8 +365,7 @@ class HyperCLOVAXOmniVideoProcessor(BaseVideoProcessor):
 
         size = SizeDict(shortest_edge=min_pixels, longest_edge=max_pixels)
 
-        # --- 2. _further_process_kwargs 패턴: resample→interpolation, mean/std→tuple ---
-        from transformers.image_processing_utils_fast import pil_torch_interpolation_mapping
+        # 2. Convert resample -> interpolation, mean/std -> tuple
         if isinstance(resample, (PILImageResampling, int)):
             interpolation = pil_torch_interpolation_mapping[resample]
         else:
@@ -360,18 +376,17 @@ class HyperCLOVAXOmniVideoProcessor(BaseVideoProcessor):
         if isinstance(image_std, list):
             image_std = tuple(image_std)
 
-        # --- 3. 비디오별 처리 ---
+        # 3. Per-video processing
         pixel_values_list = []
         grid_thw_list = []
         num_video_tokens_list = []
 
         for video in videos:
-            # _prepare_input_videos 패턴: numpy(NHWC) → channel-first torch(NCHW)
+            # numpy (NHWC) -> channel-first torch (NCHW)
             if isinstance(video, np.ndarray):
                 video = to_channel_dimension_format(video, ChannelDimension.FIRST)
                 video = torch.from_numpy(video).contiguous()
 
-            # convert_to_rgb: BaseVideoProcessor.convert_to_rgb (torchvision 기반)
             if do_convert_rgb:
                 video = self.convert_to_rgb(video)
 
@@ -406,15 +421,34 @@ class HyperCLOVAXOmniVideoProcessor(BaseVideoProcessor):
         image_width: Optional[int] = None,
         image_height: Optional[int] = None,
         num_frames: Optional[int] = None,
-        pixel_values_videos: Optional["torch.Tensor"] = None,
+        pixel_values_videos: Optional[torch.Tensor] = None,
         include_boundary_tokens: bool = False,
         patch_size: Optional[int] = None,
         temporal_patch_size: Optional[int] = None,
         merge_size: Optional[int] = None,
         min_pixels: Optional[int] = None,
         max_pixels: Optional[int] = None,
-    ) -> tuple:
-        """비디오 입력에 대한 (continuous, discrete) 토큰 수를 계산합니다."""
+        return_tuple: Optional[bool] = None,
+    ) -> int:
+        """Compute the number of video tokens for the given input.
+
+        Args:
+            image_width: Frame width (used when pixel_values_videos is None).
+            image_height: Frame height (used when pixel_values_videos is None).
+            num_frames: Number of frames (used when pixel_values_videos is None).
+            pixel_values_videos: Pre-computed pixel values tensor.
+            include_boundary_tokens: Whether to include start/end boundary tokens.
+            patch_size: ViT patch size. Defaults to self.video_patch_size.
+            temporal_patch_size: Temporal patch size. Defaults to self.video_temporal_patch_size.
+            merge_size: Token reduction merge size. Defaults to self.video_merge_size.
+            min_pixels: Minimum pixel count. Defaults to self.size["shortest_edge"].
+            max_pixels: Maximum pixel count. Defaults to self.size["longest_edge"].
+            return_tuple: If True, return (continuous, discrete) tuple.
+                Otherwise return the sum.
+
+        Returns:
+            Token count as int, or (continuous, discrete) tuple if return_tuple is True.
+        """
         patch_size = patch_size if patch_size is not None else self.video_patch_size
         temporal_patch_size = temporal_patch_size if temporal_patch_size is not None else self.video_temporal_patch_size
         merge_size = merge_size if merge_size is not None else self.video_merge_size
@@ -440,4 +474,7 @@ class HyperCLOVAXOmniVideoProcessor(BaseVideoProcessor):
         if include_boundary_tokens:
             num_continuous_tokens += 2
 
-        return (num_continuous_tokens, num_discrete_tokens)
+        if return_tuple:
+            return (num_continuous_tokens, num_discrete_tokens)
+        else:
+            return num_continuous_tokens + num_discrete_tokens