
    (HJj                     ^   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZmZmZ ddlmZ ddlmZ e G d d	e             Z G d
 dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Zy)    )	dataclass)AnyDictOptionalUnionN   )BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_rope)	SwitchGLUc                   &   e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zeed<   dZ	e
e   ed<   dZe
e   ed<   dZe
e   ed<   d
Zeed<   d
Zeed<   dZeed<   d
Zeed<   dZe
eeeeef   f      ed<   dZeed<   d Zy)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizenum_expertsnum_experts_per_tokFnorm_topk_probNhead_dimmax_position_embeddingsnum_key_value_headsattention_biasmlp_biasi'  
rope_thetarope_traditionalrope_scalingTtie_word_embeddingsc                 @    | j                   | j                  | _         y y N)r   r   selfs    ]/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/olmoe.py__post_init__zModelArgs.__post_init__$   s!    ##+'+'?'?D$ ,    )__name__
__module____qualname__str__annotations__intfloatr   boolr   r   r   r   r   r   r   r    r!   r   r   r"   r(    r)   r'   r   r      s    OO ND "Hhsm"-1Xc]1)-#- ND HdJ"d";?L(4U5#:%6 678? $$@r)   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 ^   t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  xs |j                  |z  x| _        }|dz  | _        t        j                  |||z  |j                        | _        t        j                  |||z  |j                        | _        t        j                  |||z  |j                        | _        t        j                  ||z  ||j                        | _        t!        | j                  |j"                  |j$                  |j&                  |j(                        | _        t        j,                  ||z  |j.                        | _        t        j,                  ||z  |j.                        | _        y )Ng      ࿩bias)super__init__r   r   n_headsr   
n_kv_headsr   scalennLinearr   q_projk_projv_projo_projr   r   r    r!   r   ropeRMSNormr   q_normk_norm)r&   r5   dimr;   r<   r   	__class__s         r'   r:   zAttention.__init__*   sU   !%!9!99w'+'?'??*#'==#OD4D4D4OOt^
iiWx%7d>Q>QRiiZ(%:ATATUiiZ(%:ATATUii( 2Cd>Q>QR#MMOO!!((
	 jj8!3T5F5FGjjh!68I8IJr)   xmaskcachereturnc                 v   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}| j	                  |      }| j                  |      }|j                  ||| j                  d      j                  dddd      }|j                  ||| j                  d      j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j                  ||d      }
| j                  |
      S )Nr      r      )offset)rL   r=   rK   )shaper@   rA   rB   rF   rG   reshaper;   	transposer<   rD   rR   update_and_fetchr   r=   rC   )r&   rJ   rK   rL   BLDquerieskeysvaluesoutputs              r'   __call__zAttention.__call__E   s    ''1a $AAAv++g&{{4 //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r)   NNr*   r+   r,   r   r:   mxarrayr   r   r^   __classcell__rI   s   @r'   r4   r4   )   sY    KY K< $(#	#88# rxx # }	#
 
#r)   r4   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )OlmoeSparseMoeBlockr5   c                 b   t         |           |j                  | _        |j                  | _        |j
                  | _        t        j                  |j                  | j                  d      | _	        t        |j                  |j                  | j                  |j                        | _        y NFr7   )r9   r:   r   r   top_kr   r>   r?   r   gater   r   r   
switch_mlpr&   r5   rI   s     r'   r:   zOlmoeSparseMoeBlock.__init__a   s    ++--
"11IId..0@0@uM	#""	
r)   rJ   rM   c                 
   |j                   \  }}}|j                  d|      }| j                  |      }t        j                  |dd      }| j
                  }t        j                  t        j                  | |dz
  d      dd |f         }	t        j                  ||	d      }
| j                  r|
|
j                  dd      z  }
| j                  ||	      }||
d	   z  j                  d
      }|j                  |||      S )NrO   r   T)axisprecise)kthrn   .)rn   )rn   keepdims).N)rS   rT   rj   ra   softmaxri   stop_gradientargpartitiontake_along_axisr   sumrk   )r&   rJ   rW   rX   rY   x_flatrouter_logitsrouting_weightskindicesscoresys               r'   r^   zOlmoeSparseMoeBlock.__call__o   s    ''1a2q!		&)**]DIJJ""OO_,!a%bA#rr'J
 ##OW2Ffjjb4j@@FOOFG,	""''R'0yyAq!!r)   	r*   r+   r,   r   r:   ra   rb   r^   rc   rd   s   @r'   rf   rf   `   s*    
Y 
""(( "rxx "r)   rf   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr5   c                 "   t         |           t        |      | _        t	        |      | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y )Neps)r9   r:   r4   	self_attnrf   mlpr>   rE   r   r   input_layernormpost_attention_layernormrl   s     r'   r:   zTransformerBlock.__init__   sf    "4&t,!zz$*:*:@Q@QR(*

$"3"3)
%r)   rJ   rK   rL   rM   c                     || j                  | j                  |      ||      z   }|| j                  | j                  |            z   }|S r$   )r   r   r   r   )r&   rJ   rK   rL   s       r'   r^   zTransformerBlock.__call__   sI     t33A6eDD66q9::r)   r_   r`   rd   s   @r'   r   r      sW    
Y 
 $(#	88 rxx  }	
 
r)   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )
OlmoeModelr5   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        j                  |j                  |j                        | _        y c c}w )Nr   )r5   r   )r9   r:   r5   r   r   r>   	Embeddingr   embed_tokensranger   layersrE   r   norm)r&   r5   _rI   s      r'   r:   zOlmoeModel.__init__   s    	//!%!7!7"""LL$:J:JK16t7M7M1N
1NA$'1N
 JJt//T5F5FG	
s   Cinputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rL   )r   lenr   r
   zipr   )r&   r   rL   hrK   layercs          r'   r^   zOlmoeModel.__call__   ss    
 f%=FS--E$Qa1DKK/HE1aQ'A 0yy|r)   r$   r   rd   s   @r'   r   r      s'    
HY 
H r)   r   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr5   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y rh   )r9   r:   r5   r   r   modelr"   r>   r?   r   r   lm_headrl   s     r'   r:   zModel.__init__   sX    	//%
''99T%5%5tUSDL (r)   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r$   )r   r5   r"   r   	as_linearr   )r&   r   rL   outs       r'   r^   zModel.__call__   sW    
 jj'99((**))33C8C 
 ,,s#C
r)   c                 p   d|vr|S t        | j                  j                        D ]  }d| }dD ]|  }dD ]u  }| d| d| |v st        | j                  j                        D cg c]  }|j	                  | d| d| d|         }}t        j                  |      || d| d| <   w ~  |S c c}w )	Nz+model.layers.0.mlp.experts.0.up_proj.weightzmodel.layers.)up_proj	down_proj	gate_proj)weightscalesbiasesz.mlp.experts.0..z.mlp.experts.z.mlp.switch_mlp.)r   r5   r   r   popra   stack)r&   weightslprefixnr{   eto_joins           r'   sanitizezModel.sanitize   s    8GNtyy223A$QC(F:7A 1QC8GC &+499+@+@%A#%A $KK6(-s!A3as(KL%A   # GIhhwFW6(*:1#Qqc BC 8 ; 4 #s   &#B3
c                 .    | j                   j                  S r$   )r   r   r%   s    r'   r   zModel.layers   s    zz   r)   r$   )r*   r+   r,   r   r:   ra   rb   r^   r   propertyr   rc   rd   s   @r'   r   r      s@    TY T 

 ! !r)   r   )dataclassesr   typingr   r   r   r   mlx.corecorera   mlx.nnr>   baser	   r
   r   
rope_utilsr   switch_layersr   r   Moduler4   rf   r   r   r   r2   r)   r'   <module>r      s    " - -   T T ' $ @ @ @44#		 4#n"")) "@ryy * 6&!BII &!r)   