
    (HJj                     J   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZ e G d de             Z G d	 d
e	j"                        Z G d de	j"                        Z G d de	j"                        Z G d de	j"                        Z G d de	j"                        Zy)    )	dataclass)AnyOptionalN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   y)	ModelArgshidden_sizenum_hidden_layersintermediate_sizenum_attention_headsnum_key_value_headsrms_norm_eps
vocab_sizeattention_biashead_dimmax_position_embeddingsmlp_bias
model_type
rope_thetatie_word_embeddingsN)__name__
__module____qualname__int__annotations__floatboolstr     ^/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/helium.pyr   r      sR    OM  NOr$   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	HeliumAttentionargsc                 ^   t         |           |j                  }|j                  x| _        }|j
                  J |j
                  x| _        }|j                  |z  }|dz  | _        t        j                  |||z  |j                        | _        t        j                  |||z  |j                        | _        t        j                  |||z  |j                        | _        t        j                  ||z  |d      | _        t        j                  |d|j                         | _        y )Ng      ࿩biasFT)traditionalbase)super__init__r   r   n_headsr   
n_kv_headsscalennLinearr   q_projk_projv_projo_projRoPEr   rope)selfr(   dimr0   r1   r   	__class__s         r%   r/   zHeliumAttention.__init__    s    !%!9!99w''333'+'?'??*##w.t^
iiWx%7d>Q>QRiiZ(%:ATATUiiZ(%:ATATUii( 2CeDGGH$T__M	r$   xmaskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r      )offset)r@   r2   r?   )shaper5   r6   r7   reshaper0   	transposer1   r:   rF   update_and_fetchr
   r2   r8   )r;   r>   r?   r@   BLDquerieskeysvaluesoutputs              r%   __call__zHeliumAttention.__call__1   si    ''1a $AAAv //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r$   NNr   r   r   r   r/   mxarrayr   r   rR   __classcell__r=   s   @r%   r'   r'      sY    NY N( $(#	#88# rxx # }	#
 
#r$   r'   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )	HeliumMLPr(   c                    t         |           |j                  | _        |j                  | _        t	        j
                  | j                  | j                  |j                        | _        t	        j
                  | j                  | j                  |j                        | _        t	        j
                  | j                  | j                  |j                        | _	        y )Nr*   )
r.   r/   r   r   r3   r4   r   	gate_projup_proj	down_projr;   r(   r=   s     r%   r/   zHeliumMLP.__init__P   s    ++!%!7!7d444==
 yyd444==
 ""D$4$44==
r$   r>   rA   c                 t    | j                  t        | j                  |      | j                  |                  S N)r^   r   r\   r]   )r;   r>   s     r%   rR   zHeliumMLP.__call___   s)    ~~fT^^A%6QHIIr$   	r   r   r   r   r/   rU   rV   rR   rW   rX   s   @r%   rZ   rZ   O   s-    
Y 
J"(( Jrxx Jr$   rZ   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	HeliumDecoderLayerr(   c                 D   t         |           |j                  | _        t        |      | _        t        |      | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y )Neps)r.   r/   r   r'   	self_attnrZ   mlpr3   RMSNormr   input_layernormpost_attention_layernormr_   s     r%   r/   zHeliumDecoderLayer.__init__d   st    ++(.T?!zz$*:*:@Q@QR(*

$"3"3)
%r$   r>   r?   r@   rA   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S ra   )rh   rk   ri   rl   )r;   r>   r?   r@   rhouts          r%   rR   zHeliumDecoderLayer.__call__o   sQ     NN4//2D%@EHHT22156!e
r$   rS   rT   rX   s   @r%   rd   rd   c   sW    	
Y 	
 $(#	
88
 rxx 
 }	

 

r$   rd   c                   `     e Zd Zdef fdZ	 ddej                  dej                  fdZ xZS )HeliumModelr(   c                    t         |           |j                  | _        |j                  | _        | j                  dkD  sJ t	        j
                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _
        t	        j                  |j                  |j                        | _        y c c}w )Nr   rf   )r.   r/   r   r   r3   	Embeddingr   embed_tokensrangerd   layersrj   r   norm)r;   r(   _r=   s      r%   r/   zHeliumModel.__init__}   s    !%!7!7//"""LL$:J:JK9>t?U?U9VW9VA)$/9VWJJt//T5F5FG	 Xs   	CinputsrA   c                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )ru   lenrw   r	   ziprx   )r;   rz   r@   ro   r?   layercs          r%   rR   zHeliumModel.__call__   su    
 f%=FS--E$Qa1DKK/HE1aq!A 0 yy|r$   ra   rb   rX   s   @r%   rr   rr   |   s5    
HY 
H  
	r$   rr   c                   p     e Zd Zdef fdZ	 ddej                  dej                  fdZed        Z	 xZ
S )Modelr(   c                 r   t         |           || _        |j                  | _        t	        |      | _        |j                  | _        t        j                  |j                  |j                  d      | _
        |j                  s2t        j                  |j                  |j                  d      | _
        y y )NFr*   )r.   r/   r(   r   rr   modelr   r3   r4   r   lm_headr   r_   s     r%   r/   zModel.__init__   s    	// &
//yy!1!14??O''99T%5%5tUSDL (r$   rz   rA   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S ra   )r   r(   r   ru   	as_linearr   )r;   rz   r@   rp   s       r%   rR   zModel.__call__   sW    
 jj'99((**))33C8C 
 ,,s#C
r$   c                 .    | j                   j                  S ra   )r   rw   )r;   s    r%   rw   zModel.layers   s    zz   r$   ra   )r   r   r   r   r/   rU   rV   rR   propertyrw   rW   rX   s   @r%   r   r      sI    TY T  

 
	
 ! !r$   r   )dataclassesr   typingr   r   mlx.corecorerU   mlx.nnr3   activationsr   r-   r   r	   r
   r   Moduler'   rZ   rd   rr   r   r#   r$   r%   <module>r      s    "      T T   "-#bii -#`J		 J( 2")) >!BII !r$   