
    (HJjX:                        d dl mZ d dlmZmZmZ d dlmZ d dl	m
Z
 d dlmZmZmZ ddlmZ ddlmZmZmZ ddlmZmZ dd	lmZ dd
lmZ e G d de             Zej:                  d        Z G d de
j>                        Z  G d de
j>                        Z! G d de
j>                        Z" G d de
j>                        Z# G d de
j>                        Z$ G d de
j>                        Z% G d de
j>                        Z&y)    )	dataclass)AnyListOptionalN)shard_inplaceshard_linearsum_gradients   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)KVCacheRotatingKVCache)initialize_rope)	SwitchGLUc                   b   e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   ee   ed<   ee   ed<   dZ	eed<   dZ
eed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZee   ed <   dZee   ed!<   d"Zeed#<   d$ Zy)%	ModelArgs
model_type
vocab_sizehidden_sizeintermediate_sizemoe_intermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_headshead_dimnum_expertsnum_experts_per_toknum_shared_expertsrms_norm_epsmax_position_embeddingssliding_windowlayer_typesis_moe_layerr
   n_group
topk_groupg      @routed_scaling_factorTnorm_topk_probsigmoidscoring_funcnoaux_tctopk_methodg    .A
rope_thetaNrope_scalingrope_parametersFtie_word_embeddingsc                 d    | j                   $d| j                   v r| j                   d   | _        y y y )Nr.   )r0   r.   selfs    b/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/exaone_moe.py__post_init__zModelArgs.__post_init__/   s6    +@T@T0T"22<@DO 1U+    )__name__
__module____qualname__str__annotations__intfloatr   boolr&   r'   r(   r)   r+   r-   r.   r/   r   dictr0   r1   r6    r7   r5   r   r      s    OOM  ct*GSJ#&5&ND!L#!!K!!J!#'L(4.'&*OXd^* %%Ar7   r   c                    t        j                  | j                  t         j                              }|}||z   }|dkD  rt        j                  |d|df      }t        j
                  |dd      j                  dd      }	||z
  }
t        j                  |	|
dz
  d	      d
d |
d d f   }t        j                  |t        j                  |      t        j                  d      d      }t        j                  |dd      }|}
t        j                  | |
dz
  d	      d
d |
f   }t        j                  ||d      }|dkD  r|r|j                  dd      }||dz   z  }||z  }||fS )Nr
   )axisshape   rD   T)rD   keepdims)kthrD   .g        g#B;)mxr*   astypefloat32	unflattentopksumargpartitionput_along_axisstop_gradientarrayflattentake_along_axis)gatese_score_correction_biastop_kr&   r'   r(   r)   scoresorig_scoresgroup_scoresk	group_idxindsdenominators                 r5   group_expert_selectra   4   sZ    ZZRZZ01FK--F{f2gr]Cwwvqr266B6Nj OOLa!e"Ec2A2qjQ	""B$$Y/#R
 FB+A??F7AB7RaR@DT;Fqy^jjb4j8;./++F<r7   c                   *     e Zd Zdef fdZd Z xZS )MoEGateargsc                    t         |           |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  | j                  |j                  f      | _        t        j                  | j                  f      | _        |j                  dk(  sJ d       y )Nr,   zUnsupported topk method.)super__init__r   rY   r)   r   n_routed_expertsr(   r&   r'   rK   zerosr   weightrX   r-   r4   rd   	__class__s     r5   rg   zMoEGate.__init__Y   s    --
"11 $ 0 0%)%?%?"||//hh 5 5t7G7GHI')xx1F1F0H'I$:-I/II-r7   c           	          t        || j                  j                  z  | j                  | j                  | j
                  | j                  | j                  | j                        S N)	ra   rj   TrX   rY   r&   r'   r(   r)   r4   xs     r5   __call__zMoEGate.__call__e   sO    "((JJLLOO&&
 	
r7   r8   r9   r:   r   rg   rr   __classcell__rl   s   @r5   rc   rc   X   s    
JY 
J	
r7   rc   c                   6     e Zd Zddedee   f fdZd Z xZS )MLPrd   r   c                    t         |           |j                  }|xs |j                  }t	        j
                  ||d      | _        t	        j
                  ||d      | _        t	        j
                  ||d      | _        y NFbias)	rf   rg   r   r   nnLinear	gate_projup_proj	down_proj)r4   rd   r   r   rl   s       r5   rg   zMLP.__init__r   si    &&-G1G1G;0ANyy.?eL#4kNr7   c                 t    | j                  t        | j                  |      | j                  |                  S rn   )r   r   r~   r   rp   s     r5   rr   zMLP.__call__z   s)    ~~fT^^A%6QHIIr7   rn   )	r8   r9   r:   r   r   r=   rg   rr   rt   ru   s   @r5   rw   rw   q   s#    OY O8C= OJr7   rw   c                   *     e Zd Zdef fdZd Z xZS )MoErd   c                 <   t         |           t        |j                  |j                  |j
                        | _        t        |      | _        |j                  3|j                  dkD  r$t        ||j                  |j                  z        nd | _        d | _        y )Nr   )r   )rf   rg   r   r   r   r   
switch_mlprc   gater    rw   shared_expertssharding_grouprk   s     r5   rg   zMoE.__init__   s    #&&
 DM	 &&2t7N7NQR7R	 "&"<"<t?V?V"V
  	 #r7   c                    | j                    t        | j                         |      }| j                  |      \  }}| j                  ||      }||d   z  j	                  d      j                  |j                        }| j                  || j                  |      z   }| j                   +t        j                  j                  || j                         }|S )N).NrI   rG   group)r   r	   r   r   rP   rL   dtyper   rK   distributedall_sum)r4   rq   r_   rZ   ys        r5   rr   zMoE.__call__   s    *2d11215Ayy|fOOAt$	""''R'077@*D''**A*&&q0C0C&DAr7   rs   ru   s   @r5   r   r   ~   s    #Y #*r7   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
	Attentionrd   	layer_idxc                    t         |           |j                  | _        |j                  | _        |j
                  | _        |j                  | _        | j                  dz  | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  z  | j                  d      | _        t        j                  | j                  |j                         | _        t        j                  | j                  |j                         | _        |j&                  |   dk(  | _        d|j&                  v| _        | j(                  xs | j*                  | _        | j,                  r>t/        | j                  |j0                  d|j2                  |j4                        | _        y y )Ng      Frz   epssliding_attention)basetraditionalscaling_configr"   )rf   rg   r   r   n_headsr   
n_kv_headsr   scaler|   r}   q_projk_projv_projo_projRMSNormr!   q_normk_normr$   is_sliding_windowapply_rope_all_layersuse_roper   r.   r/   r"   roper4   rd   r   rl   s      r5   rg   zAttention.__init__   s   ++//22]]D(
iidllT]]:
 iidoo=E
 iidoo=E
 iiLL4==($*:*:
 jjD4E4EFjjD4E4EF!%!1!1)!<@S!S%8@P@P%P"..L$2L2L=='__!#00(,(D(DDI r7   rq   maskcachereturnc                    |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}| j	                  |j                  ||| j                  d            j                  dddd      }| j                  |j                  ||| j                  d            j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|\| j                  r:| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n.| j                  r"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j                  ||d      }
| j!                  |
      S )NrC   r   rF   r
      )offset)r   r   r   )rE   r   r   r   r   reshaper   	transposer   r   r   r   r   update_and_fetchr   r   r   )r4   rq   r   r   BLDquerieskeysvaluesoutputs              r5   rr   zAttention.__call__   s    ''1a $AAAv++gooaDLL"EFPPq!Q
 {{4<<1doorBCMMq!Q
 1door:DDQ1aP}}))GELL)Ayyelly; 11$?LD&]]ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r7   NNr8   r9   r:   r   r=   rg   rK   rT   r   r   rr   rt   ru   s   @r5   r   r      s_    $Y $3 $R $(#	#88# rxx # }	#
 
#r7   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
DecoderLayerrd   r   c                    t         |           t        ||      | _        |j                  |   rt        |      n
t        |      | _        | j                  j                  | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y Nr   )rf   rg   r   	self_attnr%   r   rw   mlpr   r|   r   r   r!   input_layernormpost_attention_layernormr   s      r5   rg   zDecoderLayer.__init__   s    "43 $ 1 1) <3t9#d)!%!A!A!zz$*:*:@Q@QR(*

$"3"3)
%r7   rq   r   r   r   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   S rn   )r   r   r   r   )r4   rq   r   r   rhs         r5   rr   zDecoderLayer.__call__   sL     NN4//2D%@EHHT221561ur7   r   r   ru   s   @r5   r   r      s^    

Y 

3 

 $(#		88	 rxx 	 }		
 
	r7   r   c                   j     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	 xZ
S )ExaoneMoEModelrd   c                 v   t         |           || _        |j                  | _        t	        j
                  |j                  |j                        | _        t        |j                        D cg c]  }t        ||       c}| _        t	        j                  |j                  |j                        | _        d | _        d | _        t#        | j                        D ]E  \  }}|j$                  r| j                  || _        |j$                  r2| j                   ?|| _        G |j&                  | _        y c c}w r   )rf   rg   rd   r   r|   	Embeddingr   embed_tokensranger   r   layersr   r!   normswa_idxga_idx	enumerater   r#   window_size)r4   rd   idxilayerrl   s        r5   rg   zExaoneMoEModel.__init__  s    	//LL$:J:JK:?@V@V:WX:W3|D#.:WXJJt//T5F5FG	!$++.HAu&&4<<+? **t{{/B	 /  .. Ys   .D6inputsr   r   c                    | j                  |      }|d gt        | j                        z  }t        || j                  || j                     n|d         }t        || j
                  || j
                     n|d   | j                        }t        | j                  |      D ]  \  }}|j                  r|n|} ||||      }! | j                  |      S )Nr   )r   )
r   lenr   r   r   r   r   zipr   r   )	r4   r   r   r   global_maskswa_maskr   cr   s	            r5   rr   zExaoneMoEModel.__call__  s    
 f%=FS--E+T[[%<uT[[!%(
 )#'<<#;E$,,q((
 DKK/HE1$668KDaq!A 0 yy|r7   rn   )r8   r9   r:   r   rg   rK   rT   r   r   rr   rt   ru   s   @r5   r   r     sA    /Y /*  $ } 
	r7   r   c                        e Zd Zdef fdZ	 ddej                  dee   fdZ	d Z
ed        Zed        Zd	 Zdd
eej                  j                      fdZ xZS )Modelrd   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y ry   )rf   rg   rd   r   r   modelr1   r|   r}   r   r   lm_headrk   s     r5   rg   zModel.__init__4  sX    	//#D)
''99T%5%5tUSDL (r7   r   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S rn   )r   rd   r1   r   	as_linearr   )r4   r   r   outs       r5   rr   zModel.__call__=  sW    
 jj'99((**))33C8C 
 ,,s#C
r7   c                    |j                         D ci c]  \  }}|j                  d      r|| }}}|}t        | j                  j                        D ]  }| j                  j
                  |   sd| }| d}||v r|j                  |      || d<   dD ]  }dD ]  }| d| d| }	| d	| j                  j                  d
z
   d| d| }
|	|v s6|
|v s;t        | j                  j                        D cg c]  }|j                  | d	| d| d|         }}t        j                  |      || d| d| <      | j                  j                  r|j                  dd        |S c c}}w c c}w )Nzmtp.zmodel.layers.z.mlp.e_score_correction_biasz!.mlp.gate.e_score_correction_bias)r~   r   r   )rj   scalesbiasesz.mlp.experts.0..z.mlp.experts.r
   z.mlp.switch_mlp.zlm_head.weight)items
startswithr   rd   r   r%   popr   rK   stackr1   )r4   weightsr]   vnew_weightslprefixbias_keym	first_keylast_keyeto_joins                r5   sanitizezModel.sanitizeI  s   (/T1q||F?Sq!tTtyy223A99))!,$QC(F !=>H7"HOI6("CDE ;7A#)(/!AaS AI!(-		0E0E0I/J!A3aPQsS  !G+G0C &+499+@+@%A#%A $KK6(-s!A3as(KL%A   # GIhhwFW6(*:1#Qqc BC 8 ; 42 99((KK($/? U,#s   E1E1<#E7
c                 .    | j                   j                  S rn   )r   r   r3   s    r5   r   zModel.layersk  s    zz   r7   c                     d }|S )Nc                 
    d| vS )NrX   rA   )r]   s    r5   	predicatez'Model.cast_predicate.<locals>.predicateq  s    ,A55r7   rA   )r4   r   s     r5   cast_predicatezModel.cast_predicateo  s    	6 r7   c                     g }| j                   D ]X  }|j                  r1|j                  t        | j                  j
                  d             @|j                  t                      Z |S )Nr   )max_sizekeep)r   r   appendr   rd   r#   r   )r4   cachesr   s      r5   
make_cachezModel.make_cachev  sU    [[E&&#TYY-E-EAN gi( ! r7   r   c                 b   |xs t         j                  j                         }|j                         }| j                  j
                  D ]  }t        |j                  j                  d|      |j                  _        t        |j                  j                  d|      |j                  _	        t        |j                  j                  d|      |j                  _
        t        |j                  j                  d|      |j                  _        |j                  xj                  |z  c_        |j                  xj                  |z  c_        t        |j                  t               rt        |j                  j"                  d|      |j                  _        t        |j                  j$                  d|      |j                  _        t        |j                  j&                  d|      |j                  _        ||j                  _        |j                  j*                  t-        |j                  j*                  j"                  d|       t-        |j                  j*                  j$                  d|       t-        |j                  j*                  j&                  d|       t-        |j                  j.                  j"                  d|       t-        |j                  j.                  j$                  d|       t-        |j                  j.                  j&                  d|        y )Nzall-to-shardedr   zsharded-to-all)rK   r   initsizer   r   r   r   r   r   r   r   r   r   
isinstancer   rw   r~   r   r   r   r   r   r   )r4   r   Nr   s       r5   shardzModel.shard  s]   .,,.JJLZZ&&E%1&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" OO##)#OO&&1,&%))S)&2II'')9'		# '3II'')9'		# %1II%%'7u%		! ,1		(99++7!		00::(#
 "		00::(#
 "		0088:JRW II((224DE II((224DE II((002B%a 'r7   rn   )r8   r9   r:   r   rg   rK   rT   r   r   rr   r   propertyr   r   r  r   Groupr  rt   ru   s   @r5   r   r   3  s    TY T  $

 }
 D ! !  	68BNN$8$89 6r7   r   )'dataclassesr   typingr   r   r   mlx.corecorerK   mlx.nnr|   mlx.nn.layers.distributedr   r   r	   activationsr   r   r   r   r   r   r   r   
rope_utilsr   switch_layersr   r   compilera   Modulerc   rw   r   r   r   r   r   rA   r7   r5   <module>r     s    " & &   P P  T T + ' $ A A AD    F
bii 
2
J")) 
J#")) #LF#		 F#R299 2*RYY *ZDBII Dr7   