
    (HJj                     ^   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Zy)    )	dataclass)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   dZee   ed<   dZ	eed<   dZ
eed<   dZeeeeeef   f      ed<   dZeed<   y)	ModelArgs
model_typehidden_sizedim_model_basenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizenum_key_value_headsscale_depth	scale_embNmax_position_embeddingsg    .A
rope_thetaFrope_traditionalrope_scalingtie_word_embeddings)__name__
__module____qualname__str__annotations__intfloatr   r   r   r   boolr   r   r   r        _/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/minicpm.pyr   r      s    OO-1Xc]1!J!"d";?L(4U3:%6 678? %%r)   r   c                   $     e Zd Z fdZd Z xZS )MLPc                 H   t         |           t        j                  |j                  |j
                  d      | _        t        j                  |j                  |j
                  d      | _        t        j                  |j
                  |j                  d      | _        y NFbias)	super__init__nnLinearr   r   	gate_projup_proj	down_projselfargs	__class__s     r*   r2   zMLP.__init__#   sq    4#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXr)   c                 t    | j                  t        | j                  |      | j                  |                  S N)r7   r	   r5   r6   )r9   xs     r*   __call__zMLP.__call__)   s)    ~~fT^^A%6QHIIr)   )r    r!   r"   r2   r?   __classcell__r;   s   @r*   r,   r,   "   s    YJr)   r,   c                   r     e Zd Zdef fdZ	 	 ddej                  deej                     dee   fdZ	 xZ
S )	Attentionr:   c                    t         |           || _        |j                  | _        |j                  x| _        }|j                  | _        |j                  |z  x| _        }|dz  | _        |j                  | _	        | j
                  | j                  z  | _
        t        j                  | j                  | j
                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j
                  | j                  z  | j                  d      | _        t#        | j                  |j                  |j$                  |j&                  |j(                        | _        y )Ng      Fr/   )r1   r2   r:   r   r   	num_headsr   head_dimscaler   num_key_value_groupsr3   r4   q_projk_projv_projo_projr   r   r   r   rope)r9   r:   n_headsrF   r;   s       r*   r2   zAttention.__init__.   sp   	++#'#;#;;//#'#3#3w#>>t^
#'#;#; $(NNd6N6N$N!iidnnt}}<5
 iid66FU
 iid66FU
 iiNNT]]*D,<,<5
 $MMOO!!((
	r)   r>   maskcachec                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r      )offset)rP   rG   rO   )shaperI   rJ   rK   reshaperE   	transposer   rM   rU   update_and_fetchr   rG   rL   )r9   r>   rO   rP   BL_querieskeysvaluesattn_outputs              r*   r?   zAttention.__call__Q   sr    ''1a $AAAv//!Q;EEaAqQ||Aq$":":B?II!QPQSTU1d&>&>CMMq!Q
 iii=G99T%,,97D 11$?LD&ii(G99T?D2T6djjt
 "++Aq!Q7??1bI{{;''r)   NNr    r!   r"   r   r2   mxarrayr   r   r?   r@   rA   s   @r*   rC   rC   -   sJ    !
Y !
L $(#	(88( rxx ( }	(r)   rC   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	DecoderLayerr:   c                    t         |           || _        |j                  | _        |j                  | _        t        |      | _        t        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        |j                  | _        |j                  | _        y )Neps)r1   r2   r:   r   r   rC   	self_attnr,   mlpr3   RMSNormr   input_layernormpost_attention_layernormr   r8   s     r*   r2   zDecoderLayer.__init__s   s    	++!%!7!7"4t9!zz$*:*:@Q@QR(*

$"3"3)
%  ++!%!7!7r)   r>   rO   rP   returnc                    | j                  | j                  |      ||      }||| j                  | j                  dz  z  z  z   }| j	                  | j                  |            }||| j                  | j                  dz  z  z  z   }|S )Ng      ?)rj   rm   r   r   rk   rn   )r9   r>   rO   rP   rhouts          r*   r?   zDecoderLayer.__call__   s     NN4//2D%@T%%(>(>(CCDDHHT22156!t''$*@*@#*EEFF
r)   ra   rb   rA   s   @r*   rf   rf   r   sW    8Y 8& $(#	
88
 rxx 
 }	

 

r)   rf   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )MiniCPMModelr:   c                    t         |           || _        |j                  | _        | j                  dkD  sJ t	        j
                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t	        j                  |j                  |j                        | _        y c c}w )Nr   rh   )r1   r2   r:   r   r3   	Embeddingr   embed_tokensranger   rf   layersrl   r   norm)r9   r:   r\   r;   s      r*   r2   zMiniCPMModel.__init__   s    	//"""LL$:J:JK389O9O3PQ3Pa|D)3PQJJt//T5F5FG	 Rs   ?C
inputsc                    | j                  |      | j                  j                  z  }|d gt        | j                        z  }t        ||d         }t        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rx   r:   r   lenrz   r   zipr{   )r9   r|   rP   rr   rO   layercs          r*   r?   zMiniCPMModel.__call__   s    
 f%		(;(;;=FS--E$Qa1DKK/HE1aq!A 0 yy|r)   r=   )	r    r!   r"   r   r2   rc   rd   r?   r@   rA   s   @r*   ru   ru      s'    HY H r)   ru   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr:   c                    t         |           || _        |j                  | _        t	        |      | _        | j                  j                  s2t        j                  |j                  |j                  d      | _        y y r.   )r1   r2   r:   r   ru   modelr   r3   r4   r   r   lm_headr8   s     r*   r2   zModel.__init__   s\    	//!$'
yy,,99T%5%5tUSDL -r)   r|   c                 2   | j                  ||      }| j                  j                  sA| j                  || j                  j                  | j                  j
                  z  z        }|S || j                   j                  j                  j                  z  }|S r=   )	r   r:   r   r   r   r   rx   weightT)r9   r|   rP   rs   s       r*   r?   zModel.__call__   s{    
 jj'yy,,,,sdii&;&;dii>V>V&VWXC 
 

//66888C
r)   c                     d|vr|d   |d<   |S )Nzlm_head.weightzmodel.embed_tokens.weightr(   )r9   weightss     r*   sanitizezModel.sanitize   s     7*(/0K(LG$%r)   c                 .    | j                   j                  S r=   )r   rz   )r9   s    r*   rz   zModel.layers   s    zz   r)   r=   )r    r!   r"   r   r2   rc   rd   r?   r   propertyrz   r@   rA   s   @r*   r   r      s@    TY T 
 ! !r)   r   )dataclassesr   typingr   r   r   r   mlx.corecorerc   mlx.nnr3   activationsr	   baser
   r   r   
rope_utilsr   r   Moduler,   rC   rf   ru   r   r(   r)   r*   <module>r      s    " - -    T T ' & & &&J")) JB(		 B(J299 <299 :!BII !r)   