
    (HJj,                     Z   d dl mZ d dlmZmZmZ d dlmZ d dl	m
Z
 ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 de
j(                        Z G d de
j(                        Z G d de
j(                        Z G d de
j(                        Z G d de
j(                        Zy)    )	dataclass)AnyDictOptionalN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_ropec                   ,   e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZee   ed <   d!Zeed"<   d!Zeed#<   dZeed$<   y)%	ModelArgs	youtu_llm
model_typei  
vocab_sizei   hidden_sizei   intermediate_size    num_hidden_layers   num_attention_headsnum_key_value_headsi   kv_lora_ranki   q_lora_rank@   qk_rope_head_dim   
v_head_dimqk_nope_head_dimi   max_position_embeddingsgư>rms_norm_epsi j 
rope_thetaTrope_traditionalNrope_scalingFattention_biasmlp_biastie_word_embeddings)__name__
__module____qualname__r   str__annotations__r   intr   r   r   r   r   r   r   r   r   r   r    r!   floatr"   r#   boolr$   r   r   r%   r&   r'        a/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/youtu_llm.pyr   r      s    !J!JK!s!s!!!!L#KcJc#)S)L%J!d!#'L(4.' ND Hd $$r1   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	YoutuLLMAttentionconfigc                 l   t         |           || _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        |j                  | _
        |j                  | _        |j                  | _        |j                  |j                  z   | _        | j                  dz  | _        | j                  ?t        j                   | j                  | j
                  | j                  z  d      | _        nt        j                   | j                  | j                  |j$                        | _        t        j(                  | j                  |j*                        | _        t        j                   | j                  | j
                  | j                  z  d      | _        t        j                   | j                  | j                  | j                  z   |j$                        | _        t        j(                  | j                  |j*                        | _        t        j                   | j                  | j
                  | j                  | j                  z
  | j                  z   z  d      | _        t        j                   | j
                  | j                  z  | j                  |j$                        | _        t9        | j                  | j                  |j:                  |j<                  |j                        | _        y )Ng      Fbiaseps)basetraditionalscaling_configr    ) super__init__r5   r   r   	num_headsr    r"   r   r   r   r   r   
q_head_dimscalennLinearq_projr%   q_a_projRMSNormr!   q_a_layernormq_b_projkv_a_proj_with_mqakv_a_layernorm	kv_b_projo_projr   r#   r$   ropeselfr5   	__class__s     r2   r?   zYoutuLLMAttention.__init__'   so   !--33'-'E'E$ ++!-- & 7 7"// ++ & 7 7 11F4K4KK__d*
#))  $..4??"BDK II  $"2"29N9NDM "$D,<,<&BUBU!VDII  $..4??"BDM #%)) 5 55&&#

 !jj):):@S@STNN!6!66HJ	
 iiNNT__,&&
 $!!//!..$*$B$B
	r1   xmaskcachereturnc                    |j                   \  }}}| j                  | j                  |      }n/| j                  | j	                  | j                  |                  }|j                  ||| j                  | j                        j                  dddd      }t        j                  || j                  gd      \  }}	| j                  |      }
t        j                  |
| j                  gd      \  }
}|j                  ||d| j                        j                  dddd      }| j!                  | j#                  |
            }|j                  ||| j                  d      j                  dddd      }t        j                  || j                  gd      \  }}|| j%                  |	|j&                        }	| j%                  ||j&                        }t        j(                  || j                  d      }|j+                  t        j,                  ||gd      |      \  }}n]| j%                  |	      }	| j%                  |      }t        j(                  || j                  d      }t        j,                  ||gd      }t        j,                  ||	gd      }t/        ||||| j0                  |      }|j                  dddd      j                  ||d      }| j3                  |      S )Nr      r      )axis)rT   rB   rS   )shaper   rE   rI   rH   rF   reshaper@   rA   	transposemxsplitr   rJ   r   r   rL   rK   rN   offsetrepeatupdate_and_fetchconcatenater   rB   rM   )rP   rR   rS   rT   BLDqq_nopeq_pecompressed_kvk_pekvk_nopevalueskeysqueriesoutputs                     r2   __call__zYoutuLLMAttention.__call___   so    ''1a#AAd00q1ABCAIIaDNNDOO<FFq!QPQRxxD$9$9#:D//2 hh}t7H7H6IPRSt||Aq!T%:%:;EEaAqQ^^D//>?ZZ1dnnb1;;Aq!QG"t'<'<&=BG99T5<<0D99T5<<0D99T4>>:D 11~B7LD& 99T?D99T?D99T4>>:D>>64.r:D..&$b9-T6djjt
 !!!Q1-55aB?{{6""r1   NNr(   r)   r*   r   r?   r^   arrayr   r   rr   __classcell__rQ   s   @r2   r4   r4   &   sX    6
y 6
v $(#	+#88+# rxx +# }	+#
 
+#r1   r4   c                   D     e Zd Zdef fdZdej                  fdZ xZS )YoutuLLMMLPr5   c                    t         |           t        j                  |j                  |j
                  |j                        | _        t        j                  |j                  |j
                  |j                        | _        t        j                  |j
                  |j                  |j                        | _	        y )Nr7   )
r>   r?   rC   rD   r   r   r&   	gate_projup_proj	down_projrO   s     r2   r?   zYoutuLLMMLP.__init__   s     8 8v
 yy 8 8v
 $$f&8&8v
r1   rU   c                 t    | j                  t        | j                  |      | j                  |                  S N)r}   r   r{   r|   )rP   rR   s     r2   rr   zYoutuLLMMLP.__call__   s)    ~~fT^^A%6QHIIr1   	r(   r)   r*   r   r?   r^   ru   rr   rv   rw   s   @r2   ry   ry      s!    

y 

JRXX Jr1   ry   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	YoutuLLMDecoderLayerr5   c                 D   t         |           |j                  | _        t        |      | _        t        |      | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y )Nr9   )r>   r?   r   r4   	self_attnry   mlprC   rG   r!   input_layernormpost_attention_layernormrO   s     r2   r?   zYoutuLLMDecoderLayer.__init__   su    !--*62v&!zz&*<*<&BUBUV(*

F$7$7)
%r1   rR   rS   rT   rU   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S r   )r   r   r   r   )rP   rR   rS   rT   rhouts          r2   rr   zYoutuLLMDecoderLayer.__call__   sQ     NN4//2D%@EHHT22156!e
r1   rs   rt   rw   s   @r2   r   r      sW    
y 
 $(#	
88
 rxx 
 }	

 

r1   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )YoutuLLMModelr5   c                    t         |           || _        |j                  | _        |j                  | _        t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        j                  |j                  |j                        | _        y c c}w )N)r5   r9   )r>   r?   r5   r   r   rC   	Embeddingr   embed_tokensranger   layersrG   r!   norm)rP   r5   _rQ   s      r2   r?   zYoutuLLMModel.__init__   s     ++!'!9!9LL):):F<N<NO9>v?W?W9X
9XA /9X
 JJv11v7J7JK	
s   ?Cinputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r   lenr   r
   zipr   )rP   r   rT   r   rS   layercs          r2   rr   zYoutuLLMModel.__call__   su    
 f%=FS--E$Qa1DKK/HE1aq!A 0 yy|r1   r   r   rw   s   @r2   r   r      s'    	Ly 	L r1   r   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr5   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFr7   )r>   r?   r5   r   r   modelr'   rC   rD   r   r   lm_headrO   s     r2   r?   zModel.__init__   s]     ++"6*
))99V%7%79J9JQVWDL *r1   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r   )r   r5   r'   r   	as_linearr   )rP   r   rT   r   s       r2   rr   zModel.__call__   sW    
 jj';;****))33C8C 
 ,,s#C
r1   c                 V    | j                   j                  r|j                  dd        |S )Nzlm_head.weight)r5   r'   pop)rP   weightss     r2   sanitizezModel.sanitize   s#    ;;**KK($/r1   c                 .    | j                   j                  S r   )r   r   )rP   s    r2   r   zModel.layers   s    zz   r1   r   )r(   r)   r*   r   r?   r^   ru   rr   r   propertyr   rv   rw   s   @r2   r   r      s@    Xy X 


 ! !r1   r   )dataclassesr   typingr   r   r   mlx.corecorer^   mlx.nnrC   activationsr   r;   r	   r
   r   
rope_utilsr   r   Moduler4   ry   r   r   r   r0   r1   r2   <module>r      s    " & &    T T ' % % %.d#		 d#NJ")) J"299 0BII :!BII !r1   