
    (HJjr                        d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Zy)    )	dataclass)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zee   ed<   d
Z	ee   ed<   dZ
eed<   d
Zeeeeeef   f      ed<   dZeed<   dZeed<   dZeed<   y
)	ModelArgs
model_typehidden_size
num_layersintermediate_sizenum_attention_heads
vocab_size
rope_thetalayer_norm_epsilonnum_key_value_headsNhead_dimmax_position_embeddingsFrope_traditionalrope_scalingTtie_word_embeddingsattention_biasmlp_bias)__name__
__module____qualname__str__annotations__intfloatr   r   r   r   boolr   r   r   r   r   r        ^/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/exaone.pyr   r      s    OOO"Hhsm"-1Xc]1"d";?L(4U5#:%6 678? $$ ND Hdr)   r   c            	            e Zd Zdef fdZ	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	AttentionModuleargsc                    t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  xs ||z  x| _        }|dz  | _        t        j                  |||z  |j                        | _        t        j                  |||z  |j                        | _        t        j                  |||z  |j                        | _        t        j                  ||z  ||j                        | _        t!        | j                  |j"                  |j$                  |j&                  |j(                        | _        y )Ng      ࿩bias)super__init__r   r   n_headsr   
n_kv_headsr   scalennLinearr   q_projk_projv_projout_projr   r   r   r   r   rope)selfr-   dimr3   r4   r   	__class__s         r*   r2   zAttentionModule.__init__#   s   !%!9!99w'+'?'??*#'==#DSG^Dt^
iiWx%7d>Q>QRiiZ(%:ATATUiiZ(%:ATATU		'H"4c@S@ST#MMOO!!((
	r)   xmaskcachereturnc                 &   |j                   \  }}}| j                  |      j                  ||| j                  d      j	                  dddd      }| j                  |      j                  ||| j                  d      j	                  dddd      }| j                  |      j                  ||| j                  d      j	                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j	                  dddd      j                  |||      }
| j                  |
      S )Nr      r      )offset)rB   r5   rA   )shaper8   reshaper3   	transposer9   r4   r:   r<   rH   update_and_fetchr   r5   r;   )r=   r@   rA   rB   BLDqkvouts              r*   __call__zAttentionModule.__call__8   s_    ''1aKKN""1ar:DDQ1aPKKN""1a"=GG1aQRSKKN""1a"=GG1aQRS		!ELL	1A		!ELL	1A))!Q/DAq		!A		!A*q!5


 mmAq!Q'//1a8}}S!!r)   NNr    r!   r"   r   r2   mxarrayr   r   rT   __classcell__r?   s   @r*   r,   r,   "   sP    
Y 
, TX""!)"((!3"CKC="	"r)   r,   c                   $     e Zd Zdef fdZ xZS )	Attentionr-   c                 B    t         |           t        |      | _        y N)r1   r2   r,   	attentionr=   r-   r?   s     r*   r2   zAttention.__init__P   s    (.r)   )r    r!   r"   r   r2   rY   rZ   s   @r*   r\   r\   O   s    /Y / /r)   r\   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )MLPr-   c                 <   t         |           |j                  }|j                  }t	        j
                  |||j                        | _        t	        j
                  |||j                        | _        t	        j
                  |||j                        | _	        y )Nr/   )
r1   r2   r   r   r6   r7   r   c_fc_0c_fc_1c_proj)r=   r-   r>   
hidden_dimr?   s       r*   r2   zMLP.__init__V   sl    ++
iiZdmmDiiZdmmDii
CdmmDr)   r@   rC   c                 t    | j                  t        | j                  |      | j                  |                  S r^   )rf   r	   rd   re   )r=   r@   s     r*   rT   zMLP.__call__^   s(    {{6$++a.$++a.ABBr)   	r    r!   r"   r   r2   rW   rX   rT   rY   rZ   s   @r*   rb   rb   U   s/    EY EC"(( Crxx Cr)   rb   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr-   c                 "   t         |           t        j                  |j                  |j
                        | _        t        |      | _        t        j                  |j                  |j
                        | _	        t        |      | _        y N)eps)r1   r2   r6   RMSNormr   r   ln_1r\   attnln_2rb   mlpr`   s     r*   r2   zTransformerBlock.__init__c   s`    JJt//T5L5LM	dO	JJt//T5L5LM	t9r)   r@   rA   rB   rC   c                     || j                   j                  | j                  |      ||      z   }|| j                  | j	                  |            z   }|S r^   )rq   r_   rp   rs   rr   )r=   r@   rA   rB   hrS   s         r*   rT   zTransformerBlock.__call__j   sI     		##DIIaL$>>$((499Q<((
r)   rU   rV   rZ   s   @r*   rk   rk   b   sW    Y  $(#	88 rxx  }	
 
r)   rk   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )ExaoneModelr-   c                 L   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j
                  |j                        | _        y c c}w rm   )r1   r2   r6   	Embeddingr   r   wteranger   rk   ru   ro   r   ln_f)r=   r-   _r?   s      r*   r2   zExaoneModel.__init__v   su    <<1A1AB272HI2HQ"4(2HIJJt//T5L5LM	 Js   B!inputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rB   )rz   lenru   r   zipr|   )r=   r~   rB   ru   rA   layercs          r*   rT   zExaoneModel.__call__|   sr    
 HHV=FS[(E$Qa1DFFE*HE1aQ'A + yy|r)   r^   ri   rZ   s   @r*   rw   rw   u   s'    NY N r)   rw   c                   X     e Zd Zdef fdZ	 ddej                  fdZed        Z	 xZ
S )Modelr-   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFr/   )r1   r2   r-   r   rw   transformerr   r6   r7   r   r   lm_headr`   s     r*   r2   zModel.__init__   sY    	//&t,''99T%5%5tUSDL (r)   r~   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r^   )r   r-   r   rz   	as_linearr   )r=   r~   rB   rS   s       r*   rT   zModel.__call__   s[    
 vu-99((""&&005C 
 ,,s#C
r)   c                 .    | j                   j                  S r^   )r   ru   )r=   s    r*   layerszModel.layers   s    !!!r)   r^   )r    r!   r"   r   r2   rW   rX   rT   propertyr   rY   rZ   s   @r*   r   r      s;    TY T 

 " "r)   r   )dataclassesr   typingr   r   r   r   mlx.corecorerW   mlx.nnr6   activationsr	   baser
   r   r   
rope_utilsr   r   Moduler,   r\   rb   rk   rw   r   r(   r)   r*   <module>r      s    " - -    T T '   &*"bii *"Z/		 /
C")) 
Cryy &")) 2"BII "r)   