
    (HJjJ                         d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZ e G d de             Ze G d	 d
e             Z G d dej$                        Z G d dej$                        Zy)    )	dataclass)AnyDictOptionalUnionN   )BaseModelArgs)DeepseekV3Modelc                      e Zd ZU dZeed<   dZeed<   dZeed<   dZeed<   d	Z	eed
<   dZ
eed<   dZeed<   dZee   ed<   dZee   ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   d Zeed!<   d"Zeed#<   d"Zeed$<   d"Zeed%<   d"Zeed&<   d'Zeed(<   d)Z eed*<   d+Z!eed,<   d-Z"eed.<   dZ#e$ed/<   d0Z%eed1<   y)2TextArgsi  
vocab_sizei   hidden_sizei +  intermediate_sizei  moe_intermediate_size   num_hidden_layers    num_attention_headsnum_key_value_headsNn_shared_expertsn_routed_expertsg      ?routed_scaling_factori   kv_lora_ranki   q_lora_rank@   qk_rope_head_dim   
v_head_dimqk_nope_head_dimnoaux_tctopk_methodsigmoidscoring_funcTnorm_topk_probr   n_group
topk_groupnum_experts_per_tokmoe_layer_freqr   first_k_dense_replacei   max_position_embeddingsgư>rms_norm_epsg     @
rope_thetarope_scalingFattention_bias)&__name__
__module____qualname__r   int__annotations__r   r   r   r   r   r   r   r   r   r   floatr   r   r   r   r   r!   strr#   r$   boolr%   r&   r'   r(   r)   r*   r+   r,   r-   r   r.        _/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/kimi_vl.pyr   r      s/   JK"s"!%3%s!!!!&*hsm*&*hsm*#&5&L#KcJc!K!!L#!NDGSJ  NC!"3"#'S'L%JL$ ND r8   r   c                   2    e Zd ZU eeef   ed<   eed<   d Zy)	ModelArgstext_config
model_typec                 L    t         j                  | j                        | _        y N)r   	from_dictr<   selfs    r9   __post_init__zModelArgs.__post_init__2   s    #--d.>.>?r8   N)	r/   r0   r1   r   r   dictr3   r5   rC   r7   r8   r9   r;   r;   -   s    x~&&O@r8   r;   c                   R     e Zd Zdef fdZ	 ddej                  dee   fdZ	 xZ
S )LanguageModelconfigc                     t         |           || _        t        |      | _        t        j                  |j                  |j                  d      | _	        y )NF)bias)
super__init__argsr
   modelnnLinearr   r   lm_headrB   rG   	__class__s     r9   rK   zLanguageModel.__init__7   sA    	$V,
yy!3!3V5F5FUSr8   inputscachec                 H    | j                  ||      }| j                  |      S r?   )rM   rP   )rB   rS   rT   outs       r9   __call__zLanguageModel.__call__=   s#    
 jj'||C  r8   r?   )r/   r0   r1   r   rK   mxarrayr   r   rW   __classcell__rR   s   @r9   rF   rF   6   s5    Tx T  $!! }!r8   rF   c                   x     e Zd Zdef fdZ	 d	dej                  dee   fdZ	d Z
ed        Zed        Z xZS )
ModelrG   c                     t         |           || _        |j                  | _        t	        |j
                        | _        y r?   )rJ   rK   rL   r=   rF   r<   language_modelrQ   s     r9   rK   zModel.__init__G   s5    	 +++F,>,>?r8   rS   rT   c                 &    | j                  ||      S r?   )r_   )rB   rS   rT   s      r9   rW   zModel.__call__M   s    
 ""6511r8   c                    d }|j                         D ci c]  \  }} ||      s|| }}}t        | j                  j                  j                        D ]  }d| }dD ]  }dD ]  }| d| d| |v st        | j                  j                  j
                        D cg c]  }|j                  | d| d| d|         }	}t        j                  |	      || d| d| <      |S c c}}w c c}w )	Nc                 "    d| vxr
 d| vxr d| vS )Nvision_tower
rotary_embmulti_modal_projectorr7   )keys    r9   keepzModel.sanitize.<locals>.keepU   s)    c) 7 +7+36r8   zlanguage_model.model.layers.)	gate_proj	down_projup_proj)weightscalesbiasesz.mlp.experts.0..z.mlp.experts.z.mlp.switch_mlp.)	itemsrangerL   r<   r   r   poprX   stack)
rB   weightsrg   kvlprefixmeto_joins
             r9   sanitizezModel.sanitizeT   s   	 %,MMO?ODAqtAw1a4O?tyy,,>>?A3A37F@7A 1QC8GC &+499+@+@+Q+Q%R#%R $KK6(-s!A3as(KL%R   # GIhhwFW6(*:1#Qqc BC 8 A @  @#s   C0C0##C6
c                 B    | j                   j                  j                  S r?   )r_   rM   layersrA   s    r9   r}   zModel.layersk   s    ""((///r8   c                     d }|S )Nc                 
    d| vS )Ne_score_correction_biasr7   )rt   s    r9   	predicatez'Model.cast_predicate.<locals>.predicateq   s    ,A55r8   r7   )rB   r   s     r9   cast_predicatezModel.cast_predicateo   s    	6 r8   r?   )r/   r0   r1   r;   rK   rX   rY   r   r   rW   r{   propertyr}   r   rZ   r[   s   @r9   r]   r]   F   sb    @y @  $22 }2. 0 0  r8   r]   )dataclassesr   typingr   r   r   r   mlx.corecorerX   mlx.nnrN   baser	   deepseek_v3r
   r   r;   ModulerF   r]   r7   r8   r9   <module>r      su    " - -    ( !} ! !> @ @ @!BII ! .BII .r8   