
    (HJjDH                        d dl mZmZ d dlmZmZmZmZmZ d dl	m
Z d dlmZ d dlmZmZmZ d dlmZ ddlmZmZmZ ddlmZmZ dd	lmZ dd
lmZ ddlm Z! ddlm"Z# ddlm$Z% e G d de             Z& G d dejN                        Z( G d dejN                        Z) G d dejN                        Z* G d dejN                        Z+e G d de             Z, G d dejN                        Z-y)    )	dataclassfield)AnyDictListOptionalUnionN)shard_inplaceshard_linearsum_gradients)tree_map   )BaseModelArgscreate_attention_maskcreate_ssm_mask)ArraysCacheKVCache)gated_delta_update)Qwen3NextAttention)Qwen3NextMLP)Qwen3NextRMSNormGated)Qwen3NextSparseMoeBlockc            	          e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   dZ
eed	<   d
Zeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZee   ed <   dZeed!<   d"Zeed#<   d"Zeed$<   d%Zeed&<   d"Zeed'<   d"Zeed(<   d)Z eed*<    e!d+ ,      Z"ee#ee$eeee%e   f   f      ed-<   d.Z&eed/<   d0Z'eed1<   dZ(ee#ee$eef   f      ed2<   d3 Z)y)4TextModelArgs 
model_typei   hidden_sizei 8  intermediate_size    num_hidden_layersnum_attention_headsư>rms_norm_epsiQ 
vocab_size   num_key_value_headsi   max_position_embeddings@   linear_num_value_heads   linear_num_key_heads   linear_key_head_dim   linear_value_head_dim   linear_conv_kernel_dimFtie_word_embeddingsattention_biasNhead_dimfull_attention_intervalr   num_expertsnum_experts_per_tokr   decoder_sparse_stepshared_expert_intermediate_sizemoe_intermediate_sizeTnorm_topk_probc                      dg ddddS )Ndefault)   r>   
   i       ?)typemrope_section
rope_thetapartial_rotary_factor rE       _/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/qwen3_5.py<lambda>zTextModelArgs.<lambda>7   s    ) %)	!
rF   )default_factoryrope_parametersr@   rD        j@rC   rope_scalingc                    | j                   | j                  | j                  z  | _         | j                  rd| j                  vr6d| j                  v r(| j                  j	                  d      | j                  d<   | j                  j                  dd      | _        | j                  j                  dd      | _        | j                  | _        y y )NrA   	rope_typerD   r@   rC   rK   )	r4   r   r!   rJ   popgetrD   rC   rL   selfs    rG   __post_init__zTextModelArgs.__post_init__D   s    ==  ,,0H0HHDMd2224#7#77/3/C/C/G/G/T$$V,)-)=)=)A)A'*D& #2266|XNDO $ 4 4D  rF   )*__name__
__module____qualname__r   str__annotations__r   intr   r    r!   r#   floatr$   r&   r'   r)   r+   r-   r/   r1   r2   boolr3   r4   r   r5   r6   r7   r8   r9   r:   r;   r   rJ   r   r	   r   rD   rC   rL   rS   rE   rF   rG   r   r      s   JK"s"s!!L%J  #)S)"$C$ "#"""!$3$"#C# %% ND "Hhsm"#$S$ K    +,#S,!"3"ND PU
POXd3eS$S	.I(J#JKL  $(5' J ;?L(4U5#:%6 678?5rF   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	GatedDeltaNetconfigc                    t         |           |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _
        | j                  | j                  z  | _        | j                  | j                  z  | _        | j                  | j                  z  dk7  r&t        d| j                   d| j                   d      |j                  | _        |j                   | _        | j                  dz  | j                  z   | _        t'        j(                  | j$                  | j$                  d| j                  | j$                  d      | _        t'        j,                  | j                  | j                  dz  | j                  z   d      | _        t'        j,                  | j                  | j                  d      | _        t'        j,                  | j                  | j                  d      | _        t'        j,                  | j                  | j                  d      | _        t7        j8                  | j                        | _        t6        j<                  j?                  dd	| j                  f
      }t7        j@                  |      | _!        tE        | j                  | j"                        | _#        t'        j,                  | j                  | j                  d      | _$        d | _%        y )Nr   znum_v_heads (z$) must be divisible by num_k_heads ()   F)in_channelsout_channelsbiaskernel_sizegroupspaddingrd   r*   )lowhighshapeeps)&super__init__r   r)   num_v_headsr+   num_k_headsr-   
head_k_dimr/   
head_v_dimkey_dim	value_dim
ValueErrorr1   conv_kernel_sizer#   layer_norm_epsilonconv_dimnnConv1dconv1dLinearin_proj_qkv	in_proj_z	in_proj_b	in_proj_amxonesdt_biasrandomuniformlogA_logRMSNormGatednormout_projsharding_group)rR   r^   A	__class__s      rG   ro   zGatedDeltaNet.__init__W   sK   !--!88!66 44 66)9)994+;+;;d...!3 0 011UVZVfVfUgghi  !' = ="("5"5q(4>>9ii--==
 99dllQ.?e
 4#3#3T^^%P4#3#3T5E5EER4#3#3T5E5EERwwt//0II!"T5E5E4GHVVAY
 d6M6MN			$..$2B2BO"rF   inputsmaskcachereturnc                 l   |j                   \  }}}| j                   t        | j                        |      }| j                  |      }| j	                  |      j                  ||| j                  | j                        }| j                  |      }	| j                  |      }
||d   |d   }n;t        j                  || j                  dz
  | j                  f|j                        }|t        j                  |d   |d      }t        j                   ||gd      }|| j                  dz
  }|j"                  _t        j$                  |j"                  d|      }|d d d f   t        j&                  |      z   d   }t        j(                  ||d      |d<   n%t        j*                  |d d | d d d f         |d<   t-        j.                  | j1                  |            }t3        t        j4                  || j6                  d| j6                  z  gd      | j8                  | j8                  | j                  g| j:                  | j:                  | j                  g      D cg c]  \  }}}|j                  ||||       c}}}\  }}}|r|d   nd }|j                   d   dz  }|dz  t        j<                  j?                  |d d	      z  }|t        j<                  j?                  |d d	      z  }tA        ||||
|	| jB                  | jD                  ||| jF                   

      \  }}|||d<   |jI                  |       | jK                  ||      }| jM                  |j                  ||d            }| j                  +t        jN                  jQ                  || j                        }|S c c}}}w )Nr   r   )dtype).Naxisra   g      r"   )
use_kernelgroup))rk   r   r   r~   r   reshaperp   rs   r   r   r   zerosrw   ry   r   whereconcatenatelengthscliparangetake_along_axis
contiguousrz   silur|   zipsplitrt   rq   rr   fastrms_normr   r   r   trainingadvancer   r   distributedall_sum)rR   r   r   r   BS_qkvzba
conv_state
conv_inputn_keepends	positionsconv_outthdqkvstate	inv_scaleouts                             rG   __call__zGatedDeltaNet.__call__   sY    ,,1a*7]4#6#67?Fv&NN6"**1a1A1A4??SNN6"NN6"q!5qJD))A-t}}=llJ
 ((4	?C3C^^Z$5A>
**Q.F}}(wwu}}a3!!T']RYYv->>	J	--j)!La==AwxN)CDa774;;z23 DLL!dll2B#CRH!!4#3#3T5E5EF$//4??C
1a IIaAq!
1a "atGGBK4'	\RWW--at<<((D$77'JJLL==(

U E!HMM!iiQmmCKK1b12*..((D4G4G(HC
K
s   %N/NNrT   rU   rV   r   ro   r   arrayr   r   r   __classcell__r   s   @rG   r]   r]   V   s]    +#} +#` $(#	JJ rxx J }	J
 
JrF   r]   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
DecoderLayerargs	layer_idxc                    t         |           |dz   |j                  z  dk7  | _        | j                  rt	        |      | _        nt        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        |j                  dkD  rt        |      | _        y t#        |j                  |j$                        | _        y )Nr   r   rl   )rn   ro   r5   	is_linearr]   linear_attn	Attention	self_attnrz   RMSNormr   r#   input_layernormpost_attention_layernormr6   SparseMoeBlockmlpMLPr   )rR   r   r   r   s      rG   ro   zDecoderLayer.__init__   s    #a-4+G+GG1L>>,T2D&t_DN!zz$*:*:@Q@QR(*

$"3"3)
% a%d+DH4++T-C-CDDHrF   xr   r   r   c                     | j                   r#| j                  | j                  |      ||      }n"| j                  | j                  |      ||      }||z   }|| j	                  | j                  |            z   }|S N)r   r   r   r   r   r   )rR   r   r   r   rr   r   s          rG   r   zDecoderLayer.__call__   ss     >>  !5!5a!8$FAt33A6eDAE$((488;<<
rF   r   )rT   rU   rV   r   rY   ro   r   r   r   r   r   r   r   s   @rG   r   r      sa    E] Es E* $(#	88 rxx  }	
 
rF   r   c            	            e Zd Zdef fdZ	 	 ddej                  dee   deej                     dej                  fdZ	 xZ
S )	Qwen3_5TextModelr   c                    t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        ||       c}| _
        t        j                  |j
                  |j                        | _        d| _        |j                  dz
  | _        y c c}w )N)r   r   rl   r   r   )rn   ro   rz   	Embeddingr$   r   embed_tokensranger    r   layersr   r#   r   ssm_idxr5   fa_idx)rR   r   ir   s      rG   ro   zQwen3_5TextModel.__init__   s    LL$:J:JK:?@V@V:W
:WQLda0:W
 JJt//T5F5FG	22Q6
s   B>r   r   input_embeddingsr   c                 \   ||}n| j                  |      }|d gt        | j                        z  }t        ||| j                           }t        ||| j                           }t        | j                  |      D ]   \  }}|j                  r|n|}	 |||	|      }" | j                  |      S )N)r   r   )
r   lenr   r   r   r   r   r   r   r   )
rR   r   r   r   hidden_statesfa_maskssm_masklayercr   s
             rG   r   zQwen3_5TextModel.__call__   s     ',M --f5M=FS--E'uT[[7IJ"=%2EFDKK/HE1$8GD!-d!DM 0 yy''rF   r   r   r   s   @rG   r   r      sW    7] 7  $/3	(( }( #288,	(
 
(rF   r   c            	            e Zd Zdef fdZ	 	 ddej                  dee   deej                     dej                  fdZ	e
d        Zd	 Zd
 Ze
d        Ze
d        Z xZS )	TextModelr   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFrh   )rn   ro   r   r   r   modelr2   rz   r}   r   r$   lm_headrR   r   r   s     rG   ro   zTextModel.__init__  sX    	//%d+
''99T%5%5tUSDL (rF   r   r   r   r   c                     | j                  |||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S )N)r   )r   r   r2   r   	as_linearr   )rR   r   r   r   r   s        rG   r   zTextModel.__call__  s\     jj9IjJ99((**))33C8C 
 ,,s#C
rF   c                 .    | j                   j                  S r   )r   r   rQ   s    rG   r   zTextModel.layers,  s    zz   rF   c                     | j                   D cg c]$  }|j                  rt        d      n	t               & c}S c c}w )Nra   )size)r   r   r   r   )rR   ls     rG   
make_cachezTextModel.make_cache0  s0    KO;;W;aq{{#	A;WWWs   );c                    t        d |D              }t        d |j                         D              }|xs |}|j                         D ci c]  \  }}d|vs|| }}}| j                  j                  r|j	                  dd        d}|j                         D ]`  \  }dv r'|j
                  d   dk7  r|j                  d	d      |<   |s4t        fd
|D              sI|j                  dk(  sY|dz   |<   b |S c c}}w )Nc              3   $   K   | ]  }d |v  
 yw)mtp.NrE   ).0r   s     rG   	<genexpr>z%TextModel.sanitize.<locals>.<genexpr>4  s     ;7afk7s   c              3   R   K   | ]  \  }}d |v xr |j                   d   dk7   ! yw)conv1d.weightr   r   N)rk   )r   r   r   s      rG   r   z%TextModel.sanitize.<locals>.<genexpr>5  s2      %
BQ$!QOq 5QWWR[A%55/s   %'r   zlm_head.weight)z.input_layernorm.weightz .post_attention_layernorm.weightzmodel.norm.weightz.q_norm.weightz.k_norm.weightr  r   r   ra   c              3   @   K   | ]  }j                  |        y wr   endswith)r   sfxr   s     rG   r   z%TextModel.sanitize.<locals>.<genexpr>H  s     0VISCIs   g      ?)anyitemsr   r2   rO   rk   moveaxisndim)rR   weightshas_mtp_weightshas_unsanitized_conv1dshould_shift_norm_weightsr   r   	norm_keyss        `  rG   sanitizezTextModel.sanitize3  s    ;7;;!$ %
BI--/%
 "
 %4$M7M!$+MMOGODAqvQ1a4OG99((KK($/
	 MMODAq!#q(8ZZ1-
(S0VI0V-V66Q;!"SGAJ $ % Hs   DDc                 @    | j                   j                  dk  ry d }|S )Nr   c                 R    | j                  d      s| j                  d      rdddS y)Nzmlp.gateshared_expert_gater(   r%   )
group_sizebitsTr  )pathr   s     rG   	predicatez,TextModel.quant_predicate.<locals>.predicateR  s(    }}Z(DMM:N,O&(!44rF   )r   r6   rR   r  s     rG   quant_predicatezTextModel.quant_predicateM  s$    99  A%	
 rF   c                     dt         fd}|S )Nr  c                 (    | j                  d      ryy)Nr   FTr  )r  s    rG   r  z+TextModel.cast_predicate.<locals>.predicate[  s    }}W%rF   )rW   r  s     rG   cast_predicatezTextModel.cast_predicateY  s    	C 	
 rF   r   )rT   rU   rV   r   ro   r   r   r   r   r   propertyr   r   r  r  r  r   r   s   @rG   r   r     s    T] T  $/3	 } #288,	
 
 ! !X4 	 	  rF   r   c                   >     e Zd ZU eed<   eed<   e fd       Z xZS )	ModelArgsr   text_configc                 D    d|vr | |d   |      S t         |   |      S )Nr  r   )r   r  )rn   	from_dict)clsparamsr   s     rG   r!  zModelArgs.from_dicth  s-    &&"6FKKw ((rF   )	rT   rU   rV   rW   rX   dictclassmethodr!  r   r   s   @rG   r  r  c  s     O) )rF   r  c                        e Zd Zdef fdZ	 	 ddej                  deej                     fdZd Z	ddZ
ed        Zd	 Zed
        Zed        Z xZS )Modelr   c                     t         |           || _        |j                  | _        t	        t
        j                  |j                              | _        y r   )	rn   ro   r   r   r   r   r!  r  language_modelr   s     rG   ro   zModel.__init__p  s>    	//'(?(?@P@P(QRrF   r   r   c                 *    | j                  |||      S )N)r   r   )r)  )rR   r   r   r   s       rG   r   zModel.__call__v  s$     ""%2B # 
 	
rF   c                 V   i }|j                         D ]z  \  }}|j                  d      s|j                  d      r)|j                  d      r;|j                  d      r|j                  dd      }n|j                  d      rnd|z   }|||<   | | j                  j	                  |      S )Nvision_towerzmodel.visualzmodel.language_modelzlanguage_model.modelzlanguage_model.)r  
startswithreplacer)  r  )rR   r
  	sanitizedkeyvalues        rG   r  zModel.sanitize  s    	!--/JC~~n-1O~~n-~~45kk"8:PQ 12'#-"IcN * ""++I66rF   c                    |xs t         j                  j                         }|j                         |j	                         }d }fd}| j
                  D ]  }|j                  rc|j                  j                  }||j                  _	        t        |j                  j                   ||      |       |j                  j                  xj                  z  c_        t        |j                  j                  d|d|z  g|       t        |j                  j                  d|       t        |j                  j                  d|       t        |j                  j                   d|       t        j"                  t        j$                  |j                  j&                        |         |j                  _        t        j"                  t        j$                  |j                  j(                        |         |j                  _        t        |j                  j*                  d|       |j                  xj,                  z  c_        |j                  xj.                  z  c_        |j                  xj                  z  c_        |j                  xj0                  z  c_        |j                  xj2                  z  c_        nwt5        |j6                  j8                  d|      |j6                  _        t5        |j6                  j:                  d|      |j6                  _         ||j6                  j<                  |j6                  j>                          ||j6                  j@                  |j6                  j>                         t5        |j6                  j<                  d|      |j6                  _        t5        |j6                  j@                  d|      |j6                  _         |j6                  xjB                  z  c_!        tE        d|j6                  j>                  z        |j6                  _        tG        |jH                  tJ              rt5        |jH                  jL                  d|      |jH                  _&        t5        |jH                  jN                  d|      |jH                  _'        t5        |jH                  jP                  d|      |jH                  _(        ||jH                  _	        t        |jH                  jR                  jL                  d|       t        |jH                  jR                  jN                  d|       t        |jH                  jR                  jP                  d|       t        |jH                  jT                  jL                  d|       t        |jH                  jT                  jN                  d|       t        |jH                  jT                  jP                  d|        y )	Nc                       fdS )Nc                     ddz  gfS )Nr   ra   rE   )pwrt   s     rG   rH   z4Model.shard.<locals>.conv_sharding.<locals>.<lambda>  s    Wa'k$: ;rF   rE   )rt   s   `rG   conv_shardingz"Model.shard.<locals>.conv_sharding  s	    ;;rF   c                 r    k  ry fd}| j                  t        || j                                      y )Nc                     | j                   } | j                  |d   z  g|dd   } t        j                  | z  d      }  | j                  dg|dd   } | S )Nr   r   r   r   )rk   r   r   repeat)r5  sNr   s     rG   _repeatz=Model.shard.<locals>.repeat_kv_layer_inplace.<locals>._repeat  sg    GGAIIa13QqrU3IIaaa0AIIb)1QR5)rF   )updater   
parameters)r   r   r=  r<  s    ` rG   repeat_kv_layer_inplacez,Model.shard.<locals>.repeat_kv_layer_inplace  s0    Av LL'5+;+;+=>?rF   r   zall-to-shardedra   )segmentsr   zsharded-to-allr   )+r   r   initr   rankr   r   r   rt   r   r
   r|   rf   r~   r   r   r   r   r   r   r   r   rq   rp   ru   ry   r   r   o_projq_projk_projr&   v_projr!   max
isinstancer   r   	gate_proj	down_projup_projshared_expert
switch_mlp)rR   r   rC  r7  r@  r   kdr<  s          @rG   shardzModel.shard  s   .,,.JJLzz|	<	@ [[E&&..38!!0e//66b8IQVW!!((//A5/%%11$ !b&\	 %%//1A %%//1A %%//1A -/MMHHU..66:4@-!!) +---HHU..44a8>+!!' e//88:JRWX!!--!3-!!--!3-!!))a/)!!++1+!!**q0* *6OO**,<E*& *6OO**,<E*& (OO**EOO,O,O (OO**EOO,O,O *6OO**,<E*& *6OO**,<E*& 339369u::a?73
 %))S)&2II'')9'		# '3II'')9'		# %1II%%'7u%		! ,1		(II++557Gu II++557Gu II++335EU II((224DE II((224DE II((002B%u !rF   c                 B    | j                   j                  j                  S r   )r)  r   r   rQ   s    rG   r   zModel.layers  s    ""((///rF   c                 6    | j                   j                         S r   )r)  r   rQ   s    rG   r   zModel.make_cache
  s    ""--//rF   c                 .    | j                   j                  S r   )r)  r  rQ   s    rG   r  zModel.quant_predicate  s    ""222rF   c                 .    | j                   j                  S r   )r)  r  rQ   s    rG   r  zModel.cast_predicate  s    ""111rF   r   r   )rT   rU   rV   r  ro   r   r   r   r   r  rP  r  r   r   r  r  r   r   s   @rG   r'  r'  o  s    SY S /3	

 #288,	
7 tl 0 00 3 3 2 2rF   r'  ).dataclassesr   r   typingr   r   r   r   r	   mlx.corecorer   mlx.nnrz   mlx.nn.layers.distributedr
   r   r   	mlx.utilsr   baser   r   r   r   r   r   gated_deltar   
qwen3_nextr   r   r   r   r   r   r   r   r   Moduler]   r   r   r   r  r'  rE   rF   rG   <module>r`     s    ) 3 3   P P  
 ( + 7 + = A :5M :5 :5zxBII xv299 D (ryy  (FJ		 JZ ) ) )d2BII d2rF   