
    (HJj                     j   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ d dlmZ ddlmZ ddlmZmZmZ ddlmZ e G d	 d
e             Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Zy)    )	dataclass)AnyDictOptionalUnionN)shard_linear   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   d	Zeed
<   dZeed<   dZ	e
ed<   dZeeeeeef   f      ed<   dZe
ed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizenum_key_value_headsi   max_position_embeddingsi@B 
rope_thetaFrope_traditionalNrope_scalingTtie_word_embeddings)__name__
__module____qualname__str__annotations__intfloatr   r   r   boolr   r   r   r   r        ]/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/qwen2.pyr   r      s}    OO#(S(J"d";?L(4U5#:%6 678? $$r'   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 N   t         |           |j                  }|j                  x| _        }|j
                  J |j
                  x| _        }|j                  |z  }|dz  | _        t        j                  |||z  d      | _
        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        t        ||j                  |j                   |j"                  |j$                        | _        y )Ng      TbiasF)basetraditionalscaling_configr   )super__init__r   r   n_headsr   
n_kv_headsscalennLinearq_projk_projv_projo_projr   r   r   r   r   rope)selfr+   dimr4   r5   head_dim	__class__s         r(   r3   zAttention.__init__!   s    !%!9!99w''333'+'?'??*##w.t^
iiWx%7dCiiZ(%:FiiZ(%:Fii( 2CeD#--,,$($@$@
	r'   xmaskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r	      )offset)rD   r6   rC   )shaper9   r:   r;   reshaper4   	transposer5   r=   rJ   update_and_fetchr   r6   r<   )r>   rB   rC   rD   BLDquerieskeysvaluesoutputs              r(   __call__zAttention.__call__9   si    ''1a $AAAv //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r'   NNr   r   r    r   r3   mxarrayr   r   rV   __classcell__rA   s   @r(   r*   r*       sW    
Y 
6 $(#	#88# rxx # }	#
 
#r'   r*   c                   >     e Zd Z fdZdej
                  fdZ xZS )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y NFr-   )r2   r3   r7   r8   	gate_proj	down_projup_proj)r>   r?   
hidden_dimrA   s      r(   r3   zMLP.__init__X   sJ    3
?:s?yyju=r'   rE   c                 t    | j                  t        | j                  |      | j                  |                  S N)rb   r
   ra   rc   )r>   rB   s     r(   rV   zMLP.__call__^   s)    ~~fT^^A%6QHIIr'   )r   r   r    r3   rY   rZ   rV   r[   r\   s   @r(   r^   r^   W   s    >JRXX Jr'   r^   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr+   c                    t         |           |j                  | _        |j                  | _        t	        |      | _        t        |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        || _        y )Neps)r2   r3   r   r   r*   	self_attnr^   r   mlpr7   RMSNormr   input_layernormpost_attention_layernormr+   r>   r+   rA   s     r(   r3   zTransformerBlock.__init__c   s    #'#;#; ++"4t'')?)?@!zz$*:*:@Q@QR(*

$"3"3)
% 	r'   rB   rC   rD   rE   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S rf   )rl   ro   rm   rp   )r>   rB   rC   rD   rhouts          r(   rV   zTransformerBlock.__call__o   sQ     NN4//2D%@EHHT22156!e
r'   rW   rX   r\   s   @r(   rh   rh   b   sW    
Y 
 $(#	
88
 rxx 
 }	

 

r'   rh   c                   h     e Zd Zdef fdZ	 	 ddej                  deej                     fdZ xZ	S )
Qwen2Modelr+   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        j                  |j                  |j                        | _        y c c}w )Nr   )r+   rj   )r2   r3   r+   r   r   r7   	Embeddingr   embed_tokensrangerh   layersrn   r   norm)r>   r+   _rA   s      r(   r3   zQwen2Model.__init__}   s    	//!%!7!7"""LL$:J:JK16t7M7M1N
1NA$'1N
 JJt//T5F5FG	
s   Cinputsinput_embeddingsc                     ||}n| j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rz   lenr|   r   zipr}   )r>   r   rD   r   rt   rC   layercs           r(   rV   zQwen2Model.__call__   s     ' A!!&)A=FS--E$Qa1DKK/HE1aq!A 0 yy|r'   rW   )
r   r   r    r   r3   rY   rZ   r   rV   r[   r\   s   @r(   rw   rw   |   s=    
HY 
H /3	 #288,	r'   rw   c                        e Zd Zdef fdZ	 	 d
dej                  deej                     fdZd Z	ddeej                  j                     fdZed	        Z xZS )Modelr+   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y r`   )r2   r3   r+   r   rw   modelr   r7   r8   r   r   lm_headrq   s     r(   r3   zModel.__init__   sX    	//%
''99T%5%5tUSDL (r'   r   r   c                     | j                  |||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S rf   )r   r+   r   rz   	as_linearr   )r>   r   rD   r   ru   s        r(   rV   zModel.__call__   sZ     jj(8999((**))33C8C 
 ,,s#C
r'   c                     | j                   j                  r|j                  dd        |j                         D ci c]  \  }}d|vs|| c}}S c c}}w )Nzlm_head.weightzself_attn.rotary_emb.inv_freq)r+   r   popitems)r>   weightskvs       r(   sanitizezModel.sanitize   sX    99((KK($/ %]]_
,TQ0OWX0XAqD_
 	
 
s   A	Agroupc                    |xs t         j                  j                         }|j                         }| j                  j
                  D ]  }t        |j                  j                  d|      |j                  _        t        |j                  j                  d|      |j                  _	        t        |j                  j                  d|      |j                  _
        t        |j                  j                  d|      |j                  _        |j                  xj                  |z  c_        |j                  xj                  |z  c_        t        |j                  j                  d|      |j                  _        t        |j                  j                   d|      |j                  _        t        |j                  j"                  d|      |j                  _         y )Nzall-to-sharded)r   zsharded-to-all)rY   distributedinitsizer   r|   r   rl   r9   r:   r;   r<   r4   r5   rm   ra   rb   rc   )r>   r   Nr   s       r(   shardzModel.shard   se   .,,.JJLZZ&&E%1&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" OO##)#OO&&1,& #/		##%5U#EII #/		##%5U#EII !-		!!#35!EII1 'r'   c                 .    | j                   j                  S rf   )r   r|   )r>   s    r(   r|   zModel.layers   s    zz   r'   rW   rf   )r   r   r    r   r3   rY   rZ   r   rV   r   r   Groupr   propertyr|   r[   r\   s   @r(   r   r      sq    TY T /3	 #288,	
8BNN$8$89 > ! !r'   r   )dataclassesr   typingr   r   r   r   mlx.corecorerY   mlx.nnr7   mlx.nn.layers.distributedr   activationsr
   r/   r   r   r   
rope_utilsr   r   Moduler*   r^   rh   rw   r   r&   r'   r(   <module>r      s    " - -   2  T T ' % % % 4#		 4#nJ")) Jryy 4 D?!BII ?!r'   