
    (HJj                     h   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZmZmZ e G d de             Z G d d	e	j                        Z G d
 de	j                        Z G d de	j                        Z G d de	j                        Z G d de	j                        Z G d de	j                        Zy)    )	dataclass)AnyOptionalN   )BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zeed<   dZe	ed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headshead_dimrms_norm_eps
vocab_sizenum_key_value_headsi'  
rope_thetaFrope_traditionalN)
__name__
__module____qualname__str__annotations__intfloatr   r   bool     ]/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/gemma.pyr   r      sJ    OMOJ"d"r    r   c                   0     e Zd Zddedef fdZd Z xZS )RMSNormdimsepsc                 f    t         |           t        j                  |f      | _        || _        y N)super__init__mxonesweightr%   )selfr$   r%   	__class__s      r!   r)   zRMSNorm.__init__   s'    ggtg&r    c                 r    t         j                  j                  |d| j                  z   | j                        S )Ng      ?)r*   fastrms_normr,   r%   r-   xs     r!   __call__zRMSNorm.__call__!   s(    ww3#4dhh??r    )gh㈵>)r   r   r   r   r   r)   r4   __classcell__r.   s   @r!   r#   r#      s    S u 
@r    r#   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 "   t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  x| _        }|dz  | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        t        j                  ||j                   |j"                        | _        y )Ng      Fbias)traditionalbase)r(   r)   r   r   n_headsr   
n_kv_headsr   scalennLinearq_projk_projv_projo_projRoPEr   r   rope)r-   r9   dimr?   r@   r   r.   s         r!   r)   zAttention.__init__&   s    !%!9!99w'+'?'??*#'==0t^
iiWx%7eDiiZ(%:GiiZ(%:Gii( 2CeDGG--
	r    r3   maskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r      )offset)rL   rA   rK   )shaperD   rE   rF   reshaper?   	transposer@   rI   rR   update_and_fetchr	   rA   rG   )r-   r3   rK   rL   BLDquerieskeysvaluesoutputs              r!   r4   zAttention.__call__;   si    ''1a $AAAv //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r    NNr   r   r   r   r)   r*   arrayr   r   r4   r5   r6   s   @r!   r8   r8   %   sW    
Y 
0 $(#	#88# rxx # }	#
 
#r    r8   c                   >     e Zd Z fdZdej
                  fdZ xZS )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y )NFr;   )r(   r)   rB   rC   	gate_proj	down_projup_proj)r-   rJ   
hidden_dimr.   s      r!   r)   zMLP.__init__[   sJ    3
?:s?yyju=r    rM   c                     | j                  t        j                  | j                  |            | j	                  |      z        S r'   )re   rB   gelurd   rf   r2   s     r!   r4   zMLP.__call__a   s0    ~~bggdnnQ&784<<?JKKr    )r   r   r   r)   r*   r`   r4   r5   r6   s   @r!   rb   rb   Z   s    >LRXX Lr    rb   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr9   c                 v   t         |           |j                  | _        |j                  | _        t	        |      | _        t        |j                  |j                        | _        t        |j                  |j                        | _        t        |j                  |j                        | _        || _        y )Nr%   )r(   r)   r   r   r8   	self_attnrb   r   mlpr#   r   input_layernormpost_attention_layernormr9   r-   r9   r.   s     r!   r)   zTransformerBlock.__init__f   s    #'#;#; ++"4t'')?)?@&t'7'7T=N=NO(/0@0@dFWFW(X%	r    r3   rK   rL   rM   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S r'   )rn   rp   ro   rq   )r-   r3   rK   rL   rhouts          r!   r4   zTransformerBlock.__call__p   sQ     NN4//2D%@EHHT22156!e
r    r^   r_   r6   s   @r!   rk   rk   e   sW    Y  $(#	
88
 rxx 
 }	

 

r    rk   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )
GemmaModelr9   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        |j                  |j                        | _        y c c}w )Nr   )r9   rm   )r(   r)   r9   r   r   rB   	Embeddingr   embed_tokensrangerk   layersr#   r   norm)r-   r9   _r.   s      r!   r)   zGemmaModel.__init__~   s    	//!%!7!7"""LL$:J:JK16t7M7M1N
1NA$'1N
 D,,$2C2CD	
s   Cinputsc                 "   | j                  |      }|| j                  j                  dz  z  }|d gt        | j                        z  }t        ||d         }t        | j                  |      D ]  \  }} ||||      } | j                  |      S )Ng      ?r   )r{   r9   r   lenr}   r   zipr~   )r-   r   rL   ru   rK   layercs          r!   r4   zGemmaModel.__call__   s    
 f%&&+,=FS--E$Qa1DKK/HE1aq!A 0 yy|r    r'   )	r   r   r   r   r)   r*   r`   r4   r5   r6   s   @r!   rx   rx   }   s'    
EY 
E r    rx   c                   X     e Zd Zdef fdZ	 ddej                  fdZed        Z	 xZ
S )Modelr9   c                 r    t         |           |j                  | _        t        |      | _        || _        y r'   )r(   r)   r   rx   modelr9   rr   s     r!   r)   zModel.__init__   s,    //%
	r    r   c                 t    | j                  ||      }| j                   j                  j                  |      }|S r'   )r   r{   	as_linear)r-   r   rL   rv   s       r!   r4   zModel.__call__   s3    
 jj'jj%%//4
r    c                 .    | j                   j                  S r'   )r   r}   )r-   s    r!   r}   zModel.layers   s    zz   r    r'   )r   r   r   r   r)   r*   r`   r4   propertyr}   r5   r6   s   @r!   r   r      s9    Y   ! !r    r   )dataclassesr   typingr   r   mlx.corecorer*   mlx.nnrB   r>   r   r   r	   r   Moduler#   r8   rb   rk   rx   r   r   r    r!   <module>r      s    "     T T # # #@bii @2#		 2#jL")) Lryy 0 @!BII !r    