
    (HJj                        d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZmZmZ ddlmZ ddlmZ e G d d	e             Z G d
 dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Zy)    )	dataclass)AnyDictOptionalUnionN   )BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_rope)	SwitchGLUc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZe	e
eeeef   f      ed<   dZeed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizelogits_scalingattention_multiplierembedding_multiplierresidual_multipliermax_position_embeddingsnum_key_value_headsattention_bias
rope_thetanum_local_expertsnum_experts_per_tokNrope_scalingTtie_word_embeddings)__name__
__module____qualname__str__annotations__intfloatboolr!   r   r   r   r"        b/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/granitemoe.pyr   r      s    OO  ;?L(4U5#:%6 678? $$r,   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	GraniteMoeAttentionargsc                 d   t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  |z  x| _        }|j                  | _	        |j                  }t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  ||z  ||      | _        t#        | j                  |j$                  d|j&                  |j(                        | _        y )NbiasF)super__init__r   r   n_headsr   
n_kv_headshead_dimr   scaler   nnLinearq_projk_projv_projo_projr   r   r!   r   rope)selfr0   dimr6   r7   r8   r   	__class__s          r-   r5   zGraniteMoeAttention.__init__&   s    !%!9!99w'+'?'??*#'#3#3w#>>..
,,iiWx%7nMiiZ(%:PiiZ(%:Pii( 2CnM#MMOO((
	r,   xmaskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r      )offset)rF   r9   rE   )shaper<   r=   r>   reshaper6   	transposer7   r@   rL   update_and_fetchr   r9   r?   )rA   rD   rE   rF   BLDquerieskeysvaluesoutputs              r-   __call__zGraniteMoeAttention.__call__>   sg    ''1a $AAAv//!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r,   NNr#   r$   r%   r   r5   mxarrayr   r   rX   __classcell__rC   s   @r-   r/   r/   %   sW    
Y 
6 $(#	#88# rxx # }	#
 
#r,   r/   c                   L     e Zd Zdededef fdZdej                  fdZ xZS )GraniteMoeTopKGating
input_sizenum_expertstop_kc                     t         |           || _        || _        || _        t        j                  ||d      | _        y NFr2   )r4   r5   rb   ra   rc   r:   r;   layer)rA   ra   rb   rc   rC   s       r-   r5   zGraniteMoeTopKGating.__init__]   s8    &$
YYz;UC
r,   hidden_statesc                 .   | j                  |      }t        j                  || j                   d      d| j                   d f   }t        j                  ||d      }t        j
                  |j                  t        j                        d      }||fS )NrI   )kthaxis.rj   )rf   r[   argpartitionrc   take_along_axissoftmaxastypefloat32)rA   rg   logits	top_k_idxtop_k_logitstop_k_gatess         r-   rX   zGraniteMoeTopKGating.__call__d   s}    M*OOF"E$**
	 ))&)"Ejj!4!4RZZ!@rJ+%%r,   )	r#   r$   r%   r(   r5   r[   r\   rX   r]   r^   s   @r-   r`   r`   \   s1    D3 DS D D&bhh &r,   r`   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )GraniteMoeMoEr0   c                 (   t         |           |j                  | _        |j                  | _        t        | j                  | j                  |j                        | _        t        | j                  |j                  |j                        | _
        y )N)ra   rb   rc   )r4   r5   r   ra   r   r   r   
switch_mlpr`   r    routerrA   r0   rC   s     r-   r5   zGraniteMoeMoE.__init__o   ss    **11#OOT--t/E/E
 +..**
r,   rD   rG   c                     | j                  |      \  }}| j                  ||      }||d   z  j                  d      j                  |j                        S )N).Nrk   )ry   rx   sumro   dtype)rA   rD   	token_idsgatesys        r-   rX   zGraniteMoeMoE.__call__}   sQ    ;;q>	5OOAy)E)$$))r)299!''BBr,   	r#   r$   r%   r   r5   r[   r\   rX   r]   r^   s   @r-   rv   rv   n   s-    
Y 
C"(( Crxx Cr,   rv   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	GraniteMoeDecoderLayerr0   c                 D   t         |           t        |      | _        t	        |      | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        |j                  | _        y )Neps)r4   r5   r/   	self_attnrv   block_sparse_moer:   RMSNormr   r   input_layernormpost_attention_layernormr   rz   s     r-   r5   zGraniteMoeDecoderLayer.__init__   sx    ,T2 -d 3!zz$*:*:@Q@QR(*

$"3"3)
% $(#;#; r,   rD   rE   rF   rG   c                     | j                  | j                  |      ||      }||| j                  z  z   }| j                  | j	                  |            }||| j                  z  z   }|S N)r   r   r   r   r   )rA   rD   rE   rF   rhouts          r-   rX   zGraniteMoeDecoderLayer.__call__   sk     NN4//2D%@D,,,,!!$"?"?"BC!d....
r,   rY   rZ   r^   s   @r-   r   r      sW    <Y < $(#	
88
 rxx 
 }	

 

r,   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )GraniteMoEModelr0   c                 ~   t         |           || _        t        j                  |j
                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        j                  |j                  |j                        | _        |j                  | _        y c c}w )N)r0   r   )r4   r5   r0   r:   	Embeddingr   r   embed_tokensranger   r   layersr   r   normr   )rA   r0   _rC   s      r-   r5   zGraniteMoEModel.__init__   s    	LL$:J:JK7<T=S=S7T
7T!"-7T
 JJt//T5F5FG	$($=$=!	
s   B:inputsc                    | j                  |      | j                  z  }|d gt        | j                        z  }t	        ||d         }t        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rF   )r   r   lenr   r
   zipr   )rA   r   rF   r   rE   rf   cs          r-   rX   zGraniteMoEModel.__call__   s    
 f%(A(AA=FS--E$Qa1DKK/HE1aQ'A 0 yy|r,   r   r   r^   s   @r-   r   r      s%    >Y > r,   r   c                   n     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
e	d        Z xZS )	Modelr0   c                    t         |           || _        |j                  | _        t	        |      | _        |j                  s1t        j                  |j                  |j                  d      | _        |j                  | _        y re   )r4   r5   r0   r   r   modelr"   r:   r;   r   r   lm_headr   rz   s     r-   r5   zModel.__init__   sa    	//$T*
''99T%5%5tUSDL"11r,   r   c                     | j                  ||      }| j                  j                  r&| j                   j                  j	                  |      }n| j                  |      }|| j                  z  S r   )r   r0   r"   r   	as_linearr   r   )rA   r   rF   r   s       r-   rX   zModel.__call__   sZ    
 jj'99((**))33C8C,,s#CT((((r,   c                    d|vr|S t        | j                  j                        D ]  }d| d}| d}|j                  |      }t	        j
                  |dd      \  }}|||j                  dd	      <   |||j                  dd
      <   | d}|j                  |      ||j                  dd      <    | j                  j                  r|j                  dd        |S )Nz3model.layers.0.block_sparse_moe.input_linear.weightzmodel.layers.z.block_sparse_moez.input_linear.weightrJ   r   rk   input_linearzswitch_mlp.gate_projzswitch_mlp.up_projz.output_linear.weightoutput_linearzswitch_mlp.down_projzlm_head.weight)r   r0   r   popr[   splitreplacer"   )rA   weightslprefixkeyvalue	gate_projup_projs           r-   sanitizezModel.sanitize   s    @ONtyy223A$QC'89FH01CKK$E!#%!;IwKTGCKK0FGHIPGCKK0DEFH12CLSKKMGCKK1GHI 4 99((KK($/r,   c                     d }|S )Nc                 0    | j                  d      rdddS y)Nzblock_sparse_moe.router.layer@      )
group_sizebitsT)endswith)pathr   s     r-   	predicatez(Model.quant_predicate.<locals>.predicate   s    }}<=&(!44r,   r+   )rA   r   s     r-   quant_predicatezModel.quant_predicate   s    	
 r,   c                 .    | j                   j                  S r   )r   r   )rA   s    r-   r   zModel.layers   s    zz   r,   r   )r#   r$   r%   r   r5   r[   r\   rX   r   propertyr   r   r]   r^   s   @r-   r   r      sR    2Y 2 
)
)$   ! !r,   r   )dataclassesr   typingr   r   r   r   mlx.corecorer[   mlx.nnr:   baser	   r
   r   
rope_utilsr   switch_layersr   r   Moduler/   r`   rv   r   r   r   r+   r,   r-   <module>r      s    " - -   T T ' $ % % %,4#")) 4#n&299 &$CBII C*RYY 0bii :3!BII 3!r,   