
    (HJjC                        d dl mZ d dlmZ d dlmZmZmZmZ d dl	m
Z d dlmZ d dlmZmZmZ ddlmZ ddlmZmZmZ dd	lmZmZ dd
lmZ ddlmZmZ  eej@                  d      d        Z! G d dejD                        Z#e G d de             Z$ G d dejD                        Z% G d dejD                        Z&ej@                  d        Z' G d dejD                        Z( G d dejD                        Z) G d dejD                        Z* G d dejD                        Z+ G d  d!ejD                        Z, G d" d#ejD                        Z-y)$    )	dataclass)partial)AnyDictListOptionalN)shard_inplaceshard_linearsum_gradients   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)KVCacheRotatingKVCache)initialize_rope)SwiGLU	SwitchGLUT)	shapelessc                     t        j                  t        j                  |      d |      }t        j                  | | |      } || z  S )N)a_mina_max)mxclipnnsilu)xgatelimits      _/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/step3p5.pyclamped_swiglur"      s9    772774=E:D
%u-A!8O    c                   t     e Zd Zdef fdZdej                  dej                  dej                  fdZ xZS )ClampedSwiGLUr    c                 0    t         |           || _        y N)super__init__r    )selfr    	__class__s     r!   r)   zClampedSwiGLU.__init__   s    
r#   r   r   returnc                 0    t        ||| j                        S r'   )r"   r    )r*   r   r   s      r!   __call__zClampedSwiGLU.__call__   s    atzz22r#   )	__name__
__module____qualname__floatr)   r   arrayr.   __classcell__r+   s   @r!   r%   r%      s5    e 3"(( 3"(( 3rxx 3r#   r%   c                      e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   d	Zeed
<   dZeed<   dZ	e
e   ed<   dZeed<   dZeed<   dZe
ee      ed<   dZe
ee      ed<   dZe
ee      ed<   dZe
e   ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZe
e   ed <   d!Zeed"<   dZeed#<   dZe
ee      ed$<   dZe
ee      ed%<   d&Zeed'<   y)(	ModelArgs
model_typehidden_sizenum_hidden_layers
vocab_sizenum_attention_headsnum_attention_groupshead_dimintermediate_sizeh㈵>rms_norm_epsg     @
rope_thetaNrope_scalingi   max_position_embeddingsi   sliding_windowlayer_typesyarn_only_typespartial_rotary_factorsattention_other_settingTuse_head_wise_attn_gatei   moe_num_experts   	moe_top_ki   moe_intermediate_sizeshare_expert_dimmoe_layers_enumg      @moe_router_scaling_factornorm_expert_weightswiglu_limitsswiglu_limits_sharedFtie_word_embeddings)r/   r0   r1   str__annotations__intrA   r2   rB   rC   r   r   rD   rE   rF   r   rG   rH   rI   rJ   boolrK   rM   rN   rO   rP   rQ   rR   rS   rT   rU    r#   r!   r7   r7   "   s:   OOML%J#'L(4.'#)S)NC'+K$s)$++/OXd3i(/48HT%[18.2Xd^2$(T(OSIs!%3% c %)OXc])'*u*##+/M8DK(/26(4;/6 %%r#   r7   c                   b     e Zd Zddedef fdZdej                  dej                  fdZ xZ	S )ZeroCenteredRMSNormdimsepsc                 f    t         |           t        j                  |f      | _        || _        y r'   )r(   r)   r   onesweightr^   )r*   r]   r^   r+   s      r!   r)   zZeroCenteredRMSNorm.__init__C   s'    ggtg&r#   r   r,   c                 l    t         j                  j                  || j                  | j                        S r'   )r   fastrms_normra   r^   r*   r   s     r!   r.   zZeroCenteredRMSNorm.__call__H   s#    ww4;;99r#   )r@   )
r/   r0   r1   rX   r2   r)   r   r3   r.   r4   r5   s   @r!   r\   r\   B   s1    S u 
:"(( :rxx :r#   r\   c                   h     e Zd Z	 ddededef fdZdej                  dej                  fdZ	 xZ
S )	
Step3p5MLPargsr?   swiglu_limitc                    t         |           |j                  | _        || _        t	        j
                  | j                  | j                  d      | _        t	        j
                  | j                  | j                  d      | _        t	        j
                  | j                  | j                  d      | _        |r|dkD  r|| _	        y d | _	        y )NFbiasr   )
r(   r)   r9   r?   r   Linear	gate_projup_proj	down_projr    )r*   rh   r?   ri   r+   s       r!   r)   zStep3p5MLP.__init__M   s     	++!24#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWX%1lQ6F\
D
r#   r   r,   c                    | j                   D| j                  t        | j                  |      | j	                  |      | j                               S | j                  t        | j	                  |      | j                  |                  S r'   )r    rp   r"   ro   rn   r   re   s     r!   r.   zStep3p5MLP.__call__Z   sg    ::!>>t||Aq0A4::N  ~~fT^^A%6QHIIr#   )r   )r/   r0   r1   r7   rX   r2   r)   r   r3   r.   r4   r5   s   @r!   rg   rg   L   sE    MNQQ25QEJQJ"(( Jrxx Jr#   rg   c                 4   t        j                  | j                  t         j                              }||z   }t        j                  | |dz
  d      dd |f   }t        j
                  ||d      }|r|t        j                  |dd      dz   z  }|||z  fS )	Nr   )kthaxis.ru   T)ru   keepdimsg#B;)r   sigmoidastypefloat32argpartitiontake_along_axissum)	gatesrouter_biastop_krouted_scaling_factornorm_topk_probscorescorrected_scorestopk_indicestopk_weightss	            r!   moe_gate_selectr   b   s    ZZRZZ01F+??$4#4%!)"MVeVL %%flDL#FF<b485@
 (====r#   c                   D     e Zd Zdef fdZdej                  fdZ xZS )Step3p5MoEGaterh   c                 V   t         |           |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  |j                  | j
                  d      | _        t        j                  | j
                  f      | _        y NFrk   )r(   r)   rM   r   rK   n_routed_expertsrQ   r   rR   r   r   rm   r9   r   r   zerosr   r*   rh   r+   s     r!   r)   zStep3p5MoEGate.__init__u   s}    ^^
 $ 4 4%)%C%C""55IId..0E0EER	88T%:%:$<=r#   r   c                     t        | j                  |      | j                  | j                  | j                  | j
                        S r'   )r   r   r   r   r   r   re   s     r!   r.   zStep3p5MoEGate.__call__   s<    IIaLJJ&&
 	
r#   )	r/   r0   r1   r7   r)   r   r3   r.   r4   r5   s   @r!   r   r   t   s    >Y >
"(( 
r#   r   c                   `     e Zd Zdedef fdZdej                  dej                  fdZ xZ	S )
Step3p5MoErh   	layer_idxc                    t         |           d}|j                  r+|t        |j                        k  r|j                  |   xs d}d}|j                  r+|t        |j                        k  r|j                  |   xs d}t        |      | _        |dkD  rt        |      n	t               }t        |j                  |j                  |j                  |      | _        t        ||j                  |      | _        d | _        y )Nr   )
activationr?   ri   )r(   r)   rS   lenrT   r   r   r%   r   r   r9   rN   rK   
switch_mlprg   rO   share_expertsharding_group)r*   rh   r   ri   swiglu_limit_sharedr   r+   s         r!   r)   zStep3p5MoE.__init__   s    )c$2D2D.E"E--i8=AL$$S9R9R5S)S"&";";I"F"K!"4(	4@14D]<0&(
#&&  !	
 '"33,
 #r#   r   r,   c                    | j                    t        | j                         |      }| j                  |      \  }}| j                  ||      }||d   z  j	                  d      j                  |j                        }|| j                  |      z   }| j                   +t        j                  j                  || j                         }|S )N.Nrv   group)r   r   r   r   r}   ry   dtyper   r   distributedall_sum)r*   r   r   r   routed_outputys         r!   r.   zStep3p5MoE.__call__   s    *2d11215A%)YYq\"l<8\)44SbS\VM''( 	
 D--a00*&&q0C0C&DAr#   )
r/   r0   r1   r7   rX   r)   r   r3   r.   r4   r5   s   @r!   r   r      s1    #Y #3 #:"(( rxx r#   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
Step3p5Attentionrh   r   c                    t         |           |j                  }|j                  xs g }|r||   dk(  | _        n|dz  dk(  | _        | j                  r5|j
                  r)|j
                  d   | _        |j
                  d   | _        n"|j                  | _        |j                  | _        |j                  | _
        | j                  dz  | _        t        j                  || j                  | j                  z  d      | _        t        j                  || j                  | j                  z  d      | _        t        j                  || j                  | j                  z  d      | _        t        j                  | j                  | j                  z  |d      | _        t%        | j                  |j&                  	      | _        t%        | j                  |j&                  	      | _        |j,                  | _        | j,                  r't        j                  || j                  d      | _        |j0                  }t3        |t4              r||   }d
}|j6                  r'|t9        |j6                        k  r|j6                  |   }t;        | j                  |z        }|j<                  xs g }|r||   nd}	|r|	|vrd }
n|j>                  }
tA        ||d|
|jB                        | _"        y )Nsliding_attention   r   r<   r=   g      Frk   r^   g      ?full_attention)r]   basetraditionalscaling_configrD   )#r(   r)   r9   rF   
is_slidingrI   	num_headsnum_kv_headsr<   r=   r>   scaler   rm   q_projk_projv_projo_projr\   rA   q_normk_normrJ   g_projrB   
isinstancelistrH   r   rX   rG   rC   r   rD   rope)r*   rh   r   dimrF   rB   partial_rotary_factor	rope_dimsrG   
layer_typerC   r+   s              r!   r)   zStep3p5Attention.__init__   s^   &&,"))48KKDO'!mq0DO??t;;!99:OPDN $ < <=S TD!55DN $ 9 9D]]D(
iiT^^dmm%C%PiiT%6%6%FUSiiT%6%6%FUSii >%P)$--T=N=NO)$--T=N=NO'+'C'C$''))CeDDK__
j$'#I.J #&&9s4;V;V7W+W$($?$?	$J!(==>	..4"/:[+@P
z@L,,L#'$($@$@
	r#   r   maskcacher,   c                    |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}| j	                  |j                  ||| j                  d            j                  dddd      }| j                  |j                  ||| j                  d            j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      }
| j                  r*|
t!        j"                  | j%                  |            d   z  }
| j'                  |
j                  ||d            S )	Nrs   r   r   r      )offset)r   r   r   r   )shaper   r   r   r   reshaper   	transposer   r   r   r   update_and_fetchr   r   rJ   r   rx   r   r   )r*   r   r   r   BL_querieskeysvaluesoutputs              r!   r.   zStep3p5Attention.__call__   s    ''1a $AAAv++gooaDNNBGHRRq!Q
 {{4<<1d.?.?DEOOq!Q
 1d&7&7<FFq!QPQRiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-''bjjQ8CCF{{6>>!Q344r#   NNr/   r0   r1   r7   rX   r)   r   r3   r   r   r.   r4   r5   s   @r!   r   r      s_    7
Y 7
3 7
x $(#	"588"5 rxx "5 }	"5
 
"5r#   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
Step3p5DecoderLayerrh   r   c                     t         |           t        ||      | _        | j                  j                  | _        t               }|j                  r@|j                  j                         j                  d      D ch c]  }t        |       }}nt        t        d|j                              }||v | _        | j                  rt        ||      | _        nVd}|j                  r+|t!        |j                        k  r|j                  |   xs d}t#        ||j$                  |      | _        t'        |j(                  |j*                        | _        t'        |j(                  |j*                        | _        y c c}w )N,r   r   r   r   )r(   r)   r   	self_attnr   setrP   stripsplitrX   ranger:   is_moe_layerr   mlprT   r   rg   r?   r\   r9   rA   input_layernormpost_attention_layernorm)r*   rh   r   moe_layers_idxiri   r+   s         r!   r)   zStep3p5DecoderLayer.__init__  sC   )$	:..33.2.B.B.H.H.J.P.PQT.UV.Uc!f.UNV q$*@*@!ABN%7!$	2DHL((YT=V=V9W-W#88CHq!"&"8"8)DH  3$"3"3 
 )<$"3"3)
%+ Ws   =E;r   r   r   r,   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   S )Nr   r   )r   r   r   r   )r*   r   r   r   rhs         r!   r.   zStep3p5DecoderLayer.__call__;  sN     NN4//2UNKEHHT221561ur#   r   r   r5   s   @r!   r   r     s_    
Y 
3 
H $(#		88	 rxx 	 }		
 
	r#   r   c                   p     e Zd Zdef fdZ	 ddej                  deee	      dej                  fdZ
 xZS )Step3p5Modelrh   c                 8   t         |           || _        |j                  | _        |j                  | _        t        j                  |j                  |j                        | _	        t        |j                        D cg c]  }t        ||       c}| _        t        |j                  |j                        | _        t!        d t#        | j                        D        d       | _        t!        d t#        | j                        D        d       | _        y c c}w )Nr   c              3   @   K   | ]  \  }}|j                   s|  y wr'   r   .0r   ls      r!   	<genexpr>z(Step3p5Model.__init__.<locals>.<genexpr>V  s     B141aQ\\Q1   c              3   @   K   | ]  \  }}|j                   r|  y wr'   r   r   s      r!   r   z(Step3p5Model.__init__.<locals>.<genexpr>Y  s     F141aQ1r   )r(   r)   rh   r;   r:   
num_layersr   	Embeddingr9   embed_tokensr   r   layersr\   rA   normnext	enumerate_swa_idx	_full_idx)r*   rh   r   r+   s      r!   r)   zStep3p5Model.__init__H  s    	//00LL$:J:JK #4#9#9:
:	  i0:
 ((8(8d>O>OP	B9T[[1BD
 F9T[[1F

s   ?Dr   r   r,   c                    | j                  |      }|d g| j                  z  }d }d }| j                  t        ||| j                           }| j                  /t        ||| j                     | j
                  j                        }t        | j                  |      D ]   \  }}|j                  r|n|} ||||      }" | j                  |      S )N)window_sizer   )r   r   r   r   r   rh   rE   zipr   r   r   )	r*   r   r   r   	full_maskswa_masklayercr   s	            r!   r.   zStep3p5Model.__call__\  s    
 a =FT__,E	>>%-at~~1FGI==$,5'TYY5M5MH DKK/HE1$//8YDad!,A 0 yy|r#   r'   )r/   r0   r1   r7   r)   r   r3   r   r   r   r.   r4   r5   s   @r!   r   r   G  sF    
Y 
. &*88 S	" 
	r#   r   c                        e Zd Zdef fdZ	 ddej                  deee	      fdZ
ed        Zd Zd Zed	        Zed
        Zddeej"                  j$                     fdZ xZS )Modelrh   c                     t         |           || _        |j                  | _        t	        |      | _        t        j                  |j                  |j                  d      | _
        y r   )r(   r)   rh   r8   r   modelr   rm   r9   r;   lm_headr   s     r!   r)   zModel.__init__y  sJ    	//!$'
yy!1!14??Or#   inputsr   c                 H    | j                  ||      }| j                  |      S r'   )r  r  )r*   r  r   outs       r!   r.   zModel.__call__  s#    
 jj'||C  r#   c                 .    | j                   j                  S r'   )r  r   )r*   s    r!   r   zModel.layers  s    zz   r#   c                     | j                   D cg c]8  }|j                  r t        | j                  j                        n	t               : c}S c c}w )N)max_size)r   r   r   rh   rE   r   )r*   r   s     r!   
make_cachezModel.make_cache  sU     
 % ##  )A)ABY %
 	
 
s   =Ac                   	 g d	t        	fd|D              }i }|j                         D ]  \  }}d|v rd|v rX|j                  d      }t        |      dkD  r9|d   j	                         r&t        |d         | j                  j                  k\  rg	D ]"  \  }}||v s||vs|j                  ||      } n |r|j                  d      r	d|v r|d	z   }|||<    |S )
N))z.moe.gate_proj.z.mlp.switch_mlp.gate_proj.)z.moe.up_proj.z.mlp.switch_mlp.up_proj.)z.moe.down_proj.z.mlp.switch_mlp.down_proj.)z
.moe.gate.z.mlp.gate.gate.)z.moe.router_biasz.mlp.gate.router_bias)z.share_expert.z.mlp.share_expert.c              3   F   K   | ]  }D ]  \  }}||v xr ||v   y wr'   rZ   )r   ksrcdst
remappingss       r!   r   z!Model.sanitize.<locals>.<genexpr>  s2      
/6!z83C1H%A%z%ws   !z.mtpzmodel.layers..r   z.weightr   r   )
anyitemsr   r   isdigitrX   rh   r:   replaceendswith)
r*   weights
is_vanillanew_weightsr  vpartsr  r  r  s
            @r!   sanitizezModel.sanitize  s    

  
/6
 

 MMODAq{!#u:>eAh&6&6&858}		(C(CC &S!81		#s+A '
 ajj3!EKN# $& r#   c                     d }|S )Nc                 
    d| vS )Nr   rZ   )r  s    r!   	predicatez'Model.cast_predicate.<locals>.predicate  s     ))r#   rZ   r*   r!  s     r!   cast_predicatezModel.cast_predicate  s    	* r#   c                     d }|S )Nc                     d| v rdddS y)Nzmlp.gate.gate@   rL   )
group_sizebitsTrZ   )pathr   s     r!   r!  z(Model.quant_predicate.<locals>.predicate  s    $&&(!44r#   rZ   r"  s     r!   quant_predicatezModel.quant_predicate  s    	
 r#   r   c                    |xs t         j                  j                         }|j                         }| j                  j
                  D ]  }t        |j                  j                  d|      |j                  _        t        |j                  j                  d|      |j                  _	        t        |j                  j                  d|      |j                  _
        t        |j                  j                  d|      |j                  _        |j                  xj                  |z  c_        |j                  xj                  |z  c_        |j                  j                  r1t        |j                  j                  d|      |j                  _        t!        |j"                  t$              rt        |j"                  j&                  d|      |j"                  _        t        |j"                  j(                  d|      |j"                  _        t        |j"                  j*                  d|      |j"                  _        ||j"                  _        t/        |j"                  j0                  j&                  d|       t/        |j"                  j0                  j(                  d|       t/        |j"                  j0                  j*                  d|       t/        |j"                  j2                  j&                  d|       t/        |j"                  j2                  j(                  d|       t/        |j"                  j2                  j*                  d|        y )Nzall-to-shardedr   zsharded-to-all)r   r   initsizer  r   r
   r   r   r   r   r   r   r   rJ   r   r   r   rg   rn   ro   rp   r   r	   r   r   )r*   r   Nr   s       r!   shardzModel.shard  s|   .,,.JJLZZ&&E%1&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" OO%%!+%OO((Q.(66)5OO**,<E*& %))Z0&2II'')9'		# %1II%%'7u%		! '3II'')9'		# ,1		(II**446Fe II**224DE II**446Fe II((224DE II((002B% II((224DEa 'r#   r'   )r/   r0   r1   r7   r)   r   r3   r   r   r   r.   propertyr   r  r  r#  r*  r   Groupr/  r4   r5   s   @r!   r  r  x  s    PY P &*!! S	"! ! !
"H    68BNN$8$89 6r#   r  ).dataclassesr   	functoolsr   typingr   r   r   r   mlx.corecorer   mlx.nnr   mlx.nn.layers.distributedr	   r
   r   activationsr   r   r   r   r   r   r   r   
rope_utilsr   switch_layersr   r   compiler"   Moduler%   r7   r\   rg   r   r   r   r   r   r   r  rZ   r#   r!   <module>r>     s   "  , ,   P P  T T + ' , 	t$ %3BII 3 & & &>:")) :J J, > >"
RYY 
*. .b\5ryy \5~+")) +\.299 .bHBII Hr#   