
    (HJj                     j   d dl mZ d dlmZ d dlmZmZmZmZ d dl	m
Z d dlmZ ddlmZ ddlmZmZmZ ddlmZ e G d	 d
e             Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Zy)    )	dataclass)partial)AnyDictOptionalUnionN   )XieLU)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   dZeed<   dZ	e
eeeeef   f      ed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizemlp_biasnum_attention_headsattention_biasrms_norm_eps
vocab_sizenum_key_value_headsmax_position_embeddings
rope_theta	post_normqk_normtie_word_embeddingsFrope_traditionalNrope_scaling)__name__
__module____qualname__str__annotations__intboolfloatr    r!   r   r   r        _/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/apertus.pyr   r      s    ONO  OM"d";?L(4U5#:%6 678?r+   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )
ApertusMLPargsc                 ,   t         |           t        j                  |j                  |j
                  |j                        | _        t        j                  |j
                  |j                  |j                        | _        t               | _
        y )Nbias)super__init__nnLinearr   r   r   up_proj	down_projr
   act_fnselfr/   	__class__s     r,   r4   zApertusMLP.__init__%   si    yyd444==
 ""D$4$44==
 gr+   xreturnc                 `    | j                  | j                  | j                  |                  S N)r8   r9   r7   )r;   r=   s     r,   __call__zApertusMLP.__call__/   s"    ~~dkk$,,q/:;;r+   )	r"   r#   r$   r   r4   mxarrayrA   __classcell__r<   s   @r,   r.   r.   $   s*    Y <"(( <rxx <r+   r.   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	ApertusAttentionr/   c                    t         |           |j                  | _        |j                  | _        |j                  |j                  z  | _        | j
                  dz  | _        t        j                  |j                  |j                  | j
                  z  d      | _	        t        j                  |j                  |j                  | j
                  z  d      | _
        t        j                  |j                  |j                  | j
                  z  d      | _        t        j                  |j                  | j
                  z  |j                  d      | _        t        j                  | j
                  |j                        | _        t        j                  | j
                  |j                        | _        t#        | j
                  |j$                  |j&                  |j(                  |j*                        | _        y )Ng      Fr1   eps)r3   r4   r   r   r   head_dimscaler5   r6   q_projk_projv_projo_projRMSNormr   q_normk_normr   r   r    r!   r   roper:   s     r,   r4   zApertusAttention.__init__4   sw   #'#;#; #'#;#; ((D,D,DD]]D(
iid66FU
 iid66FU
 iid66FU
 ii$$t}}4d6F6FU
 jjD4E4EFjjD4E4EF#MMOO!!((
	r+   r=   maskcacher>   c                 n   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}| j	                  |j                  ||| j                  d            j                  dddd      }| j                  |j                  ||| j                  d            j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j                  ||d      }
| j                  |
      S )Nr      r	      )offset)rV   rL   rU   )shaperM   rN   rO   rR   reshaper   	transposerS   r   rT   r[   update_and_fetchr   rL   rP   )r;   r=   rU   rV   BLDquerieskeysvaluesoutputs              r,   rA   zApertusAttention.__call__T   s    ''1a $AAAv++OOAq$":":B?

)Aq!Q
 	 {{4<<1d.F.FKLVVq!Q
 1d&>&>CMMq!Q
 iii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r+   NNr"   r#   r$   r   r4   rB   rC   r   r   rA   rD   rE   s   @r,   rG   rG   3   sX    
Y 
F $(#	#88# rxx # }	#
 
#r+   rG   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	ApertusDecoderLayerr/   c                 "   t         |           t        |      | _        t	        |      | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y )NrI   )r3   r4   rG   	self_attnr.   mlpr5   rQ   r   r   attention_layernormfeedforward_layernormr:   s     r,   r4   zApertusDecoderLayer.__init__v   sd    )$/d##%::d.>.>DDUDU#V %'ZZ0@0@dFWFW%X"r+   r=   rU   rV   r>   c                     || j                  | j                  |      ||      z   }|| j                  | j                  |            z   }|S r@   )rl   rn   rm   ro   )r;   r=   rU   rV   houts         r,   rA   zApertusDecoderLayer.__call__~   sI     t77:D%HH$((455a899
r+   rg   rh   rE   s   @r,   rj   rj   u   sY    YY Y $(#	88 rxx  }	
 
r+   rj   c                   j     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	 xZ
S )ApertusModelr/   c                 N   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j
                  |j                        | _        y c c}w )N)r/   rI   )r3   r4   r5   	Embeddingr   r   embed_tokensranger   rj   layersrQ   r   norm)r;   r/   _r<   s      r,   r4   zApertusModel.__init__   s}    LL$:J:JK49$:P:P4Q
4QqT*4Q
 JJt//T5F5FG	
s   B"inputsrV   r>   c                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rU   rV   )rw   lenry   r   ziprz   )r;   r|   rV   rq   rU   layercs          r,   rA   zApertusModel.__call__   su    
 f%=FS--E$Qa1DKK/HE1ad!,A 0 yy|r+   r@   rh   rE   s   @r,   rt   rt      sC    HY H  $ } 
	r+   rt   c                        e Zd Zdef fdZ	 d	dej                  dee   dej                  fdZ	d Z
ed        Z xZS )
Modelr/   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFr1   )r3   r4   r/   r   rt   modelr   r5   r6   r   r   lm_headr:   s     r,   r4   zModel.__init__   sX    	//!$'
''99T%5%5tUSDL (r+   r|   rV   r>   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r@   )r   r/   r   rw   	as_linearr   )r;   r|   rV   rr   s       r,   rA   zModel.__call__   sW    
 jj'99((**))33C8C 
 ,,s#C
r+   c                     |j                         D ];  \  }}|j                  d      s|j                  d      s)|j                         ||<   = | j                  j                  r|j                  dd        |S )Nalpha_palpha_nzlm_head.weight)itemsendswithsqueezer/   r   pop)r;   weightskvs       r,   sanitizezModel.sanitize   s_    MMODAqzz)$

9(=YY[
 $ 99((KK($/r+   c                 .    | j                   j                  S r@   )r   ry   )r;   s    r,   ry   zModel.layers   s    zz   r+   r@   )r"   r#   r$   r   r4   rB   rC   r   r   rA   r   propertyry   rD   rE   s   @r,   r   r      s\    TY T  $

 }
 
	
 ! !r+   r   )dataclassesr   	functoolsr   typingr   r   r   r   mlx.corecorerB   mlx.nnr5   activationsr
   baser   r   r   
rope_utilsr   r   Moduler.   rG   rj   rt   r   r*   r+   r,   <module>r      s    "  - -    T T ' @ @ @(< <?#ryy ?#D")) (299 6!BII !r+   