
    (HJjt                     ^   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Zy)    )	dataclass)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   d	Zeed
<   dZeed<   dZ	e
ed<   dZeeeeeef   f      ed<   dZe
ed<   dZeed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizenum_key_value_headsi   max_position_embeddingsg     @
rope_thetaFrope_traditionalNrope_scalingtie_word_embeddings   num_nextn_predict_layers)__name__
__module____qualname__str__annotations__intfloatr   r   r   boolr   r   r   r   r   r        \/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/mimo.pyr   r      s    OO#(S(J"d";?L(4U5#:%6 678? %%$%c%r(   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 N   t         |           |j                  }|j                  x| _        }|j
                  J |j
                  x| _        }|j                  |z  }|dz  | _        t        j                  |||z  d      | _
        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        t        ||j                  |j                   |j"                  |j$                        | _        y )Ng      TbiasF)basetraditionalscaling_configr   )super__init__r   r   n_headsr   
n_kv_headsscalennLinearq_projk_projv_projo_projr   r   r   r   r   rope)selfr,   dimr5   r6   head_dim	__class__s         r)   r4   zAttention.__init__!   s    !%!9!99w''333'+'?'??*##w.t^
iiWx%7dCiiZ(%:FiiZ(%:Fii( 2CeD#--,,$($@$@
	r(   xmaskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr   r   r      )offset)rE   r7   rD   )shaper:   r;   r<   reshaper5   	transposer6   r>   rJ   update_and_fetchr   r7   r=   )r?   rC   rD   rE   BLDquerieskeysvaluesoutputs              r)   __call__zAttention.__call__9   sg    ''1a $AAAv//!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r(   NNr   r    r!   r   r4   mxarrayr   r   rV   __classcell__rB   s   @r)   r+   r+       sW    
Y 
6 $(#	#88# rxx # }	#
 
#r(   r+   c                   >     e Zd Z fdZdej
                  fdZ xZS )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y NFr.   )r3   r4   r8   r9   	gate_proj	down_projup_proj)r?   r@   
hidden_dimrB   s      r)   r4   zMLP.__init__W   sJ    3
?:s?yyju=r(   rF   c                 t    | j                  t        | j                  |      | j                  |                  S N)rb   r	   ra   rc   )r?   rC   s     r)   rV   zMLP.__call__]   s)    ~~fT^^A%6QHIIr(   )r   r    r!   r4   rY   rZ   rV   r[   r\   s   @r)   r^   r^   V   s    >JRXX Jr(   r^   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr,   c                    t         |           |j                  | _        |j                  | _        t	        |      | _        t        |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        || _        y )Neps)r3   r4   r   r   r+   	self_attnr^   r   mlpr8   RMSNormr   input_layernormpost_attention_layernormr,   r?   r,   rB   s     r)   r4   zTransformerBlock.__init__b   s    #'#;#; ++"4t'')?)?@!zz$*:*:@Q@QR(*

$"3"3)
% 	r(   rC   rD   rE   rF   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S rf   )rl   ro   rm   rp   )r?   rC   rD   rE   rhouts          r)   rV   zTransformerBlock.__call__n   sQ     NN4//2D%@EHHT22156!e
r(   rW   rX   r\   s   @r)   rh   rh   a   sW    
Y 
 $(#	
88
 rxx 
 }	

 

r(   rh   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )	MiMoModelr,   c                    t         |           || _        |j                  | _        |j                  | _        |j
                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _	        t        |j                        D cg c]  }t        |       c}| _        t        j                  |j                  |j                        | _        y c c}w )Nr   )r,   rj   )r3   r4   r,   r   r   r   r8   	Embeddingr   embed_tokensrangerh   layersrn   r   norm)r?   r,   _rB   s      r)   r4   zMiMoModel.__init__|   s    	//!%!7!7(,(E(E%"""LL$:J:JK16t7M7M1N
1NA$'1N
 JJt//T5F5FG	
s   !C-inputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      }|S )Nr   )rz   lenr|   r   zipr}   )r?   r   rE   rt   rD   layercs          r)   rV   zMiMoModel.__call__   sy    
 f%=FS--E$Qa1DKK/HE1aq!A 0 IIaLr(   rf   )	r   r    r!   r   r4   rY   rZ   rV   r[   r\   s   @r)   rw   rw   {   s'    HY H" r(   rw   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr,   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y r`   )r3   r4   r,   r   rw   modelr   r8   r9   r   r   lm_headrq   s     r)   r4   zModel.__init__   sW    	//t_
''99T%5%5tUSDL (r(   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S rf   )r   r,   r   rz   	as_linearr   )r?   r   rE   ru   s       r)   rV   zModel.__call__   sW    
 jj'99((**))33C8C 
 ,,s#C
r(   c                     | j                   j                  r|j                  dd        |j                         D ci c]  \  }}d|vr|j	                  d      s|| c}}S c c}}w )Nzlm_head.weightzself_attn.rotary_emb.inv_freqzmodel.mtp_layers.)r,   r   popitems
startswith)r?   weightskvs       r)   sanitizezModel.sanitize   sh    99((KK($/  
'1.a7LL!45 qD'
 	
 
s   "A"c                 .    | j                   j                  S rf   )r   r|   )r?   s    r)   r|   zModel.layers   s    zz   r(   rf   )r   r    r!   r   r4   rY   rZ   rV   r   propertyr|   r[   r\   s   @r)   r   r      s@    TY T 	
 ! !r(   r   )dataclassesr   typingr   r   r   r   mlx.corecorerY   mlx.nnr8   activationsr	   r0   r
   r   r   
rope_utilsr   r   Moduler+   r^   rh   rw   r   r'   r(   r)   <module>r      s    " - -    T T ' & & &"3#		 3#lJ")) Jryy 4 		  F$!BII $!r(   