
    (HJj                     n   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZmZ ddlmZ e G d	 d
e             Z G d dej0                        Z G d dej0                        Z G d dej0                        Z G d dej0                        Z G d dej0                        Zy)    )	dataclass)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)KVCacheRotatingKVCache)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eee	eef   f   ed<   e
e   ed<   e
e   ed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizenum_key_value_headsmax_position_embeddings
rope_thetahead_dimtie_word_embeddingsrope_scalingsliding_windowsliding_window_patternN)__name__
__module____qualname__str__annotations__intfloatboolr   r   r        _/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/exaone4.pyr   r      st    OO  MsE%*--..SM!$SM)r*   r   c            	            e Zd Zdedee   f fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
	Attentionargsis_localc                    t         |           |j                  }|j                  x| _        }|j
                  J |j
                  x| _        }|j                  }|dz  | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        t        j                  ||j                         | _        t        j                  ||j                         | _        |xs d| _        |d u xs || _        | j(                  r4t+        ||j,                  d|j.                  |j0                        | _        y y )Ng      Fbiaseps)basetraditionalscaling_configr   )super__init__r   r   n_headsr   
n_kv_headsr   scalennLinearq_projk_projv_projo_projRMSNormr   q_normk_normr/   use_roper   r   r   r   rope)selfr.   r/   dimr:   r;   r   	__class__s          r+   r9   zAttention.__init__#   sK   !%!9!99w''333'+'?'??*==t^
iiWx%7eDiiZ(%:GiiZ(%:Gii( 2CeDjjt/@/@Ajjt/@/@A )E D(4H=='__!#00(,(D(DDI r*   xmaskcachereturnc                    |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}| j	                  |j                  ||| j                  d            j                  dddd      }| j                  |j                  ||| j                  d            j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|\| j                  r:| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n.| j                  r"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j                  ||d      }
| j!                  |
      S )Nr      r      )offset)rM   r<   rL   )shaper?   r@   rA   rD   reshaper:   	transposerE   r;   rF   rG   rS   update_and_fetchr   r<   rB   )rH   rK   rL   rM   BLDquerieskeysvaluesoutputs              r+   __call__zAttention.__call__@   s    ''1a $AAAv++gooaDLL"EFPPq!Q
 {{4<<1doorBCMMq!Q
 1door:DDQ1aP}}))GELL)Ayyelly; 11$?LD&]]ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r*   NN)r!   r"   r#   r   r   r(   r9   mxarrayr   r_   __classcell__rJ   s   @r+   r-   r-   "   sc    Y (4. @ $(#	#88# rxx # }	#
 
#r*   r-   c                   >     e Zd Z fdZdej
                  fdZ xZS )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y NFr1   )r8   r9   r=   r>   	gate_proj	down_projup_proj)rH   rI   
hidden_dimrJ   s      r+   r9   zMLP.__init__c   sJ    3
?:s?yyju=r*   rN   c                 t    | j                  t        | j                  |      | j                  |                  S N)rj   r	   ri   rk   )rH   rK   s     r+   r_   zMLP.__call__i   s)    ~~fT^^A%6QHIIr*   )r!   r"   r#   r9   ra   rb   r_   rc   rd   s   @r+   rf   rf   b   s    >JRXX Jr*   rf   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
TransformerBlockr.   r/   c                    t         |           |j                  | _        |j                  | _        t	        ||      | _        t        |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        || _        y )Nr3   )r8   r9   r   r   r-   	self_attnrf   r   mlpr=   rC   r   post_attention_layernormpost_feedforward_layernormr.   )rH   r.   r/   rJ   s      r+   r9   zTransformerBlock.__init__n   s    #'#;#; ++"42t'')?)?@(*

$"3"3)
% +-**$"3"3+
' 	r*   rK   rL   rM   rN   c                     | j                  |||      }|| j                  |      z   }| j                  |      }|| j                  |      z   }|S rn   )rr   rt   rs   ru   )rH   rK   rL   rM   rhouts          r+   r_   zTransformerBlock.__call__|   sR     NN1dE*--a00HHQK$11!44
r*   r`   )r!   r"   r#   r   r(   r9   ra   rb   r   r   r_   rc   rd   s   @r+   rp   rp   m   s^    Y $ " $(#	
88
 rxx 
 }	

 

r*   rp   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )ExaoneModelr.   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        |j                  }t        |j                        D cg c]%  }t        ||r||t        |      z     dk(  nd       ' c}| _        |r-|j                  d      | _        |j                  d      | _        nd | _        d| _        |j"                  | _        t        j&                  |j                  |j(                        | _        y c c}w )Nr   rY   )r.   r/   Gr3   )r8   r9   r.   r   r   r=   	Embeddingr   embed_tokensr    rangerp   lenlayersindexswa_idxfull_idxr   window_sizerC   r   norm)rH   r.   patternirJ   s       r+   r9   zExaoneModel.__init__   s   	//!%!7!7"""LL$:J:JK-- 4112

 3	 =DS\!12c9$ 3
 "==-DL#MM#.DMDLDM..JJt//T5F5FG	
s   *Einputsc                    | j                  |      }|d gt        | j                        z  }t        ||| j                           }| j
                  &t        ||| j
                     | j                        }nd }t        | j                  |      D ])  \  }}|j                  j                  r|n|} ||||      }+ | j                  |      S )N)r   )r   r   r   r   r   r   r   ziprr   r/   r   )	rH   r   rM   rx   global_maskswa_masklayercrL   s	            r+   r_   zExaoneModel.__call__   s    
 f%=FS--E+AuT]]/CD<<#,5&D4D4DH HDKK/HE1$778[Daq!A 0 yy|r*   rn   )	r!   r"   r#   r   r9   ra   rb   r_   rc   rd   s   @r+   r{   r{      s'    HY H8 r*   r{   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr.   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y rh   )r8   r9   r.   r   r{   modelr   r=   r>   r   r   lm_head)rH   r.   rJ   s     r+   r9   zModel.__init__   sX    	// &
''99T%5%5tUSDL (r*   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S rn   )r   r.   r   r   	as_linearr   )rH   r   rM   ry   s       r+   r_   zModel.__call__   sW    
 jj'99((**))33C8C 
 ,,s#C
r*   c                     | j                   D cg c]C  }|j                  j                  r!t        | j                  j
                  d      n	t               E c}S c c}w )Nr   )max_sizekeep)r   rr   r/   r   r.   r   r   )rH   ls     r+   
make_cachezModel.make_cache   s[     [[
 ! ;;''  )A)AJY !
 	
 
s   AAc                 .    | j                   j                  S rn   )r   r   )rH   s    r+   r   zModel.layers   s    zz   r*   rn   )r!   r"   r#   r   r9   ra   rb   r_   r   propertyr   rc   rd   s   @r+   r   r      s@    TY T 


 ! !r*   r   )dataclassesr   typingr   r   r   r   mlx.corecorera   mlx.nnr=   activationsr	   r5   r
   r   r   rM   r   r   
rope_utilsr   r   Moduler-   rf   rp   r{   r   r)   r*   r+   <module>r      s    " - -    T T + ' * * *$=#		 =#@J")) Jryy 8/")) /d!!BII !!r*   