
    (HJj                     R   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZmZmZ ddlmZ e G d de             Z G d	 d
ej&                        Z G d dej&                        Z G d dej&                        Z G d dej&                        Z G d dej&                        Zy)    )	dataclass)AnyDictOptionalUnionN   )BaseModelArgscreate_attention_maskscaled_dot_product_attention)	SwitchGLUc                       e Zd ZU eed<   dZeed<   dZeed<   dZeed<   dZ	eed	<   dZ
eed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeeeeeef   f      ed<   dZeed<   d Zy)	ModelArgs
model_typei }  
vocab_sizei   hidden_sizei 8  intermediate_size    num_hidden_layersnum_attention_heads   num_experts_per_tok   num_key_value_headsnum_local_expertsgh㈵>rms_norm_epsg    .A
rope_thetaFrope_traditionalNrope_scalingtie_word_embeddingsc                 @    | j                   | j                  | _         y y N)r   r   selfs    _/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/mixtral.py__post_init__zModelArgs.__post_init__   s!    ##+'+'?'?D$ ,    )__name__
__module____qualname__str__annotations__r   intr   r   r   r   r   r   r   r   floatr   r   boolr   r   r   r   r   r%    r&   r$   r   r      s    OJK"s"s!!    sL%J"d";?L(4U5#:%6 678? %%@r&   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	MixtralAttentionargsc                 t   t         |           |j                  | _        |j                  | _        | j                  | j                  z  | _        |j                  | _        |j                  | _        | j
                  dz  | _        t        j                  | j                  | j                  | j
                  z  d      | _        t        j                  | j                  | j                  | j
                  z  d      | _        t        j                  | j                  | j                  | j
                  z  d      | _        t        j                  | j                  | j
                  z  | j                  d      | _        t        j                  | j
                  |j                   |j                        | _        y )Ng      Fbias)traditionalbase)super__init__r   r   	num_headshead_dimr   r   scalennLinearq_projk_projv_projo_projRoPEr   roper#   r2   	__class__s     r$   r9   zMixtralAttention.__init__$   sC   ++11((DNN:#'#;#; //]]D(
iidnnt}}<5
 iid66FU
 iid66FU
 iiNNT]]*D,<,<5
 GGMM--
	r&   xmaskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr   r   r      )offset)rI   r<   rH   )shaper?   r@   rA   reshaper:   	transposer   rD   rN   update_and_fetchr   r<   rB   )r#   rG   rH   rI   BLDquerieskeysvaluesoutputs              r$   __call__zMixtralAttention.__call__A   st    ''1a $AAAv //!Q;EEaAqQ||Aq$":":B?II!QPQSTU1d&>&>CMMq!Q
 iii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r&   NNr'   r(   r)   r   r9   mxarrayr   r   rZ   __classcell__rF   s   @r$   r1   r1   #   sX    
Y 
@ $(#	#88# rxx # }	#
 
#r&   r1   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )MixtralSparseMoeBlockr2   c                 l   t         |           |j                  | _        |j                  | _        |j                  | _        |j                  | _        t        j                  | j                  | j                  d      | _        t        | j                  | j
                  | j                        | _        y NFr4   )r8   r9   r   
hidden_dimr   ffn_dimr   num_expertsr   r=   r>   gater   
switch_mlprE   s     r$   r9   zMixtralSparseMoeBlock.__init__b   s    **--11#'#;#;  IIdoot/?/?eL	#DOOT\\4CSCSTr&   rG   rJ   c                 `   | j                  |      }| j                  }t        j                  t        j                  | |dz
  d      dd |f         }t        j
                  ||d      }t        j                  |dd      }| j                  ||      }||d   z  j                  d	      }|S )
Nr   rL   )kthaxis.)rl   T)rl   precise).N)	rh   r   r]   stop_gradientargpartitiontake_along_axissoftmaxri   sum)r#   rG   gateskindsscoresys          r$   rZ   zMixtralSparseMoeBlock.__call__n   s    		!$$AE KCQSRSQSG TU##E4b9FT:OOAt$	""''R'0r&   )	r'   r(   r)   r   r9   r]   r^   rZ   r_   r`   s   @r$   rb   rb   a   s,    
UY 
U"(( rxx r&   rb   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	MixtralDecoderLayerr2   c                 D   t         |           |j                  | _        t        |      | _        t        |      | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y )Neps)r8   r9   r   r1   	self_attnrb   block_sparse_moer=   RMSNormr   input_layernormpost_attention_layernormrE   s     r$   r9   zMixtralDecoderLayer.__init__}   sv    ++)$/ 5d ;!zz$*:*:@Q@QR(*

$"3"3)
%r&   rG   rH   rI   rJ   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S r!   )r~   r   r   r   )r#   rG   rH   rI   rhouts          r$   rZ   zMixtralDecoderLayer.__call__   sS     NN4//2D%@E!!$"?"?"BC!e
r&   r[   r\   r`   s   @r$   rz   rz   |   sW    

Y 

 $(#	
88
 rxx 
 }	

 

r&   rz   c                        e Zd Zdef fdZ	 	 ddej                  deej                     dej                  fdZ xZ	S )MixtralModelr2   c                    t         |           |j                  | _        |j                  | _        t	        j
                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _
        t	        j                  |j                  |j                        | _        y c c}w )N)r2   r|   )r8   r9   r   r   r=   	Embeddingr   embed_tokensrangerz   layersr   r   norm)r#   r2   _rF   s      r$   r9   zMixtralModel.__init__   s    //!%!7!7LL$:J:JK49$:P:P4Q
4QqT*4Q
 JJt//T5F5FG	
s   8Cinputsinput_embeddingsrJ   c                     ||}n| j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r   lenr   r
   zipr   )r#   r   rI   r   r   rH   layercs           r$   rZ   zMixtralModel.__call__   s     ' A!!&)A=FS--E$Qa1DKK/HE1aq!A 0 yy|r&   r[   )
r'   r(   r)   r   r9   r]   r^   r   rZ   r_   r`   s   @r$   r   r      sK    	HY 	H /3	 #288,	
 
r&   r   c                        e Zd Zdef fdZ	 	 d	dej                  deej                     dej                  fdZd Z	e
d        Z xZS )
Modelr2   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y rd   )r8   r9   r2   r   r   modelr   r=   r>   r   r   lm_headrE   s     r$   r9   zModel.__init__   sX    	//!$'
''99T%5%5tUSDL (r&   r   r   rJ   c                     | j                  |||      }| j                  j                  r%| j                   j                  j	                  |      S | j                  |      S r!   )r   r2   r   r   	as_linearr   )r#   r   rI   r   r   s        r$   rZ   zModel.__call__   sN     jj(8999((::**44S99<<$$r&   c                    | j                   j                  r|j                  dd        d|vr|S t        | j                   j                        D ]  }d| }dD ]  \  }}dD ]u  }| d| d| |v st        | j                   j
                        D cg c]  }|j                  | d| d| d|         }}t        j                  |      || d	| d| <   w   |S c c}w )
Nzlm_head.weightz3model.layers.0.block_sparse_moe.experts.0.w1.weightzmodel.layers.))w1	gate_proj)w2	down_proj)w3up_proj)weightscalesbiasesz.block_sparse_moe.experts.0..z.block_sparse_moe.experts.z.block_sparse_moe.switch_mlp.)r2   r   popr   r   r   r]   stack)	r#   weightslprefixnmru   eto_joins	            r$   sanitizezModel.sanitize   s   99((KK($/@ONtyy223A$QC(FU17A !=aS!EP
 &+499+F+F%G	# &H $KK#)(*DQCq1QC P &H	   # HHW-  6(*Gs!A3 OP 8 V 4 #s   #C
c                 .    | j                   j                  S r!   )r   r   r"   s    r$   r   zModel.layers   s    zz   r&   r[   )r'   r(   r)   r   r9   r]   r^   r   rZ   r   propertyr   r_   r`   s   @r$   r   r      sd    TY T /3	
%
% #288,	
%
 

%* ! !r&   r   )dataclassesr   typingr   r   r   r   mlx.corecorer]   mlx.nnr=   r7   r	   r
   r   switch_layersr   r   Moduler1   rb   rz   r   r   r/   r&   r$   <module>r      s    " - -   T T $ @ @ @*;#ryy ;#|BII 6")) 4299 D,!BII ,!r&   