Update configuration_motif.py

configuration_motif.py

@@ -134,7 +134,6 @@ class MotifConfig(PretrainedConfig):
         sliding_window=4096,
         max_window_layers=28,
         attention_dropout=0.0,
-        multi_token_heads: Optional[int] = None,
         **kwargs,
     ):
         """
@@ -165,87 +164,14 @@ class MotifConfig(PretrainedConfig):
         self.rope_scaling = rope_scaling
         self.attention_dropout = attention_dropout
 
-        ###kwargs
-
-        # some scale factors
-
-        self.scale_emb = getattr(kwargs, "scale_emb", 1)
-        self.init_scale_o = getattr(kwargs, "init_scale_o", 1)
-
-        # muparam
-        self.hidden_states_shrink = 1 / math.sqrt(num_hidden_layers)
-        self.dim_model_base = hidden_size
-        self.dim_model_base_attn  = (hidden_size // num_attention_heads)
-        self.dim_model_base_init = hidden_size
-        self.dim_model_base_lr = getattr(kwargs, "dim_model_base_lr", hidden_size//8)
-        self.dim_model_base_lmh = 1
-        self.dim_model_base_logits = hidden_size
-
-        self.muP = getattr(kwargs, "muP", False)
-        # proxy hidden size ( following YuLan-Mini )
-        # reparameterization(wesar_weights)
-        logger.info(kwargs)
-        self.wesar_weights = getattr(kwargs, "wesar_weights", False)
-        logger.info(f'initial wesar reparameterization : {self.wesar_weights}')
-
-        # alpha (scale factor)
-        self.embed_tokens_alpha = getattr(kwargs, "embed_tokens_alpha", None)
-        self.q_proj_alpha = getattr(kwargs, "q_proj_alpha", None)
-        self.k_proj_alpha = getattr(kwargs, "k_proj_alpha", None)
-        self.v_proj_alpha = getattr(kwargs, "v_proj_alpha", None)
-        self.o_proj_alpha = getattr(kwargs, "o_proj_alpha", None)
-        self.down_proj_alpha =  getattr(kwargs, "down_proj_alpha", None)
-        self.gate_up_proj_alpha =  getattr(kwargs, "gate_up_proj_alpha", None)
-        self.input_layernorm_alpha = getattr(kwargs, "input_layernorm_alpha", None)
-        self.post_attention_layernorm_alpha =  getattr(kwargs, "post_attention_layernorm_alpha", None)
-        self.norm_alpha = getattr(kwargs, "norm_alpha", None)
-        self.lm_head_alpha =  getattr(kwargs, "lm_head_alpha", None)
-        self.use_norm_alpha =  getattr(kwargs, "use_norm_alpha", False)
-        self.use_emb_alpha =  getattr(kwargs, "use_emb_alpha", False)
-
         # Validate the correctness of rotary position embeddings parameters
         # BC: if there is a 'type' field, move it to 'rope_type'.
         if self.rope_scaling is not None and "type" in self.rope_scaling:
             self.rope_scaling["rope_type"] = self.rope_scaling["type"]
         rope_config_validation(self)
-
-        self.multi_token_heads = multi_token_heads
-        self.multi_token_config_validation()
-
-
-
-        # moe
-        self.topk_method = getattr(kwargs, "topk_method", None)
-        self.scoring_func = getattr(kwargs, "scoring_func", None)
-        self.routed_scaling_factor = getattr(kwargs, "routed_scaling_factor", None)
-        self.norm_topk_prob = getattr(kwargs, "norm_topk_prob", None)
-        self.seq_aux = getattr(kwargs, "seq_aux", None)
-        self.hidden_act_moe =  getattr(kwargs, "hidden_act_moe", None)
-
-
-        self.n_group =  getattr(kwargs, "n_group", None)
-        self.n_routed_experts = getattr(kwargs, "n_routed_experts", None)
-        self.moe_intermediate_size =  getattr(kwargs, "moe_intermediate_size", None)
-        self.topk_group = getattr(kwargs, "topk_group", None)
-
-
-        self.use_fused_mlp =  getattr(kwargs, "use_fused_mlp", None)
-        self.use_moreh_moe =  getattr(kwargs, "use_moreh_moe", False)
-        self.continual_training =  getattr(kwargs, "continual_training", False)
-
-        # external
-        self.first_expansion =  getattr(kwargs, "first_expansion", False)
-        self.moe_layer =  getattr(kwargs, "moe_layer", False)
-
-
-
+        
         super().__init__(
             tie_word_embeddings=tie_word_embeddings,
             **kwargs,
         )
         logger.info(f' kwargs : {kwargs}')
-        logger.info(f'after wesar reparameterization : {self.wesar_weights}')
-
-    def multi_token_config_validation(self):
-        if self.multi_token_heads is not None:
-            assert isinstance(self.multi_token_heads, int) and self.multi_token_heads >= 1