
    (HJj|                     :   d dl mZ d dlmZ d dlmZ ddlmZ ddl	m
Z
mZmZ e G d de
             Z G d d	ej                        Z G d
 dej                        Z G d dej                        Z G d dej                        Z G d dej                        Zy)    )	dataclassN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   dZeed<   dZeed<   dZeed<   dZ	eed	<   d
Z
eed<   dZeed<   dZeed<   dZeed<   dZeed<   dZd Zy)	ModelArgs
model_typei   hidden_size   num_attention_heads   num_hidden_layers   kv_channelsi    max_position_embeddingsgư>layer_norm_epsiloni +  intermediate_sizeTno_biasiQ 
vocab_sizeNc                 @    | j                   | j                  | _         y y N)num_key_value_headsr   selfs    \/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/qwen.py__post_init__zModelArgs.__post_init__   s!    ##+'+'?'?D$ ,    )__name__
__module____qualname__str__annotations__r   intr   r   r   r   r   floatr   r   boolr   r   r    r   r   r
   r
      sr    OK!!sK#'S' $$"s"GTJ@r   r
   c                   ,     e Zd Zdef fdZddZ xZS )	Attentionargsc                    t         |           |j                  }|j                  | _        || j                  z  }t	        j
                  |d      | _        |j                  | j                  z  }t	        j                  ||dz  d      | _	        t	        j                  |||j                         | _        |dz  | _        y )NF)traditional   Tbiasg      )super__init__r   r   nnRoPE
rotary_embr   Linearc_attnr   c_projscale)r   r+   r   hidden_size_per_attention_head	proj_size	__class__s        r   r2   zAttention.__init__    s    &&#'#;#; )48P8P)P&''"@eT$$t'?'??	iiY]FiiY=MN3T9
r   c                 &   | j                  |      }t        j                  |dd      \  }}}|j                  \  }}	}
|j	                  ||	| j
                  d      j                  dddd      }|j	                  ||	| j
                  d      j                  dddd      }|j	                  ||	| j
                  d      j                  dddd      }|P| j                  ||j                        }| j                  ||j                        }|j                  ||      \  }}n"| j                  |      }| j                  |      }t        ||||| j                  |      }|j                  dddd      j	                  ||	d      }| j                  |      S )	Nr.   )axisr      r   )offset)cacher9   mask)r7   mxsplitshapereshaper   	transposer5   rA   update_and_fetchr   r9   r8   )r   xrC   rB   qkvqkvBL_querieskeysvaluesoutputs                  r   __call__zAttention.__call__1   sp   kk!n((3+1a''1a))Aq$":":B?II!QPQSTUyyAt77<FFq!QPQR1a!9!92>HHAqRSToogelloCG??4?=D 11$?LD&oog.G??4(D-T6djjt
 !!!Q1-55aB?{{6""r   NNr    r!   r"   r
   r2   rV   __classcell__r<   s   @r   r*   r*      s    :Y :"#r   r*   c                   *     e Zd Zdef fdZd Z xZS )MLPr+   c                    t         |           t        j                  |j                  |j
                  dz  |j                         | _        t        j                  |j                  |j
                  dz  |j                         | _        t        j                  |j
                  dz  |j                  |j                         | _	        y )Nr@   r/   )
r1   r2   r3   r6   r   r   r   w1w2r8   r   r+   r<   s     r   r2   zMLP.__init__M   s    ))d449DLL@P
 ))d449DLL@P
 ii""a')9)9DLL@P
r   c                 |    | j                  |      }| j                  |      }| j                  t        ||            S r   )r^   r_   r8   r   )r   rJ   a1a2s       r   rV   zMLP.__call__Z   s1    WWQZWWQZ{{6"b>**r   rX   rZ   s   @r   r\   r\   L   s    
Y 
+r   r\   c                   ,     e Zd Zdef fdZddZ xZS )TransformerBlockr+   c                 "   t         |           t        j                  |j                  |j
                        | _        t        |      | _        t        j                  |j                  |j
                        | _	        t        |      | _        y N)eps)r1   r2   r3   RMSNormr   r   ln_1r*   attnln_2r\   mlpr`   s     r   r2   zTransformerBlock.__init__a   s`    JJt//T5L5LM	dO	JJt//T5L5LM	t9r   c                     |}| j                  |      }| j                  |||      }||z   }| j                  |      }| j                  |      }||z   }|S )N)rC   rB   )rj   rk   rl   rm   )r   rJ   rC   rB   residuals        r   rV   zTransformerBlock.__call__i   sZ    IIaLIIad%I0x<IIhHHQKLr   rW   rX   rZ   s   @r   re   re   `   s    Y 	r   re   c                   ,     e Zd Zdef fdZddZ xZS )	QwenModelr+   c                 L   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j
                  |j                        | _        y c c}w rg   )r1   r2   r3   	Embeddingr   r   wteranger   re   hri   r   ln_f)r   r+   rQ   r<   s      r   r2   zQwenModel.__init__v   sw    <<1A1AB278N8N2OP2OQ"4(2OPJJt//T5L5LM	 Qs   B!c                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rt   lenrv   r   ziprw   )r   inputsrB   rJ   rC   layercs          r   rV   zQwenModel.__call__|   sp    HHV=FS[(E$Qa1DFFE*HE1aq!A + yy|r   r   rX   rZ   s   @r   rq   rq   u   s    NY N
r   rq   c                   p     e Zd Zdef fdZ	 ddej                  dej                  fdZed        Z	 xZ
S )Modelconfigc                     t         |           |j                  | _        t        |      | _        t        j                  |j                  |j                  |j                         | _
        || _        y )Nr/   )r1   r2   r   rq   transformerr3   r6   r   r   r   lm_headr+   )r   r   r<   s     r   r2   zModel.__init__   s[     ++$V,yy 1 1FNN8J
 	r   rJ   returnc                 H    | j                  ||      }| j                  |      S r   )r   r   )r   rJ   rB   ys       r   rV   zModel.__call__   s$    
 Q&||Ar   c                 .    | j                   j                  S r   )r   rv   r   s    r   layerszModel.layers   s    !!!r   r   )r    r!   r"   r
   r2   rD   arrayrV   propertyr   rY   rZ   s   @r   r   r      sG    y  88 
	 " "r   r   )dataclassesr   mlx.corecorerD   mlx.nnr3   activationsr   baser   r   r   r
   Moduler*   r\   re   rq   r   r(   r   r   <module>r      s    "    T T @ @ @$*#		 *#Z+")) +(ryy *		 ("BII "r   