
    (HJj                     t   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZ e G d de             Z G d	 d
e	j"                        Z G d de	j"                        Z G d de	j"                        Z G d de	j"                        Z G d de	j"                        Z G d de	j"                        Zy)    )	dataclass)AnyOptionalN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   dZeed<   dZeed<   dZeed<   dZ	eed	<   dZ
eed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   y)	ModelArgs
model_typei    hidden_size(   num_hidden_layersi X  intermediate_size@   num_attention_headsnum_key_value_headsg    ^A
rope_thetai  
vocab_sizegh㈵>layer_norm_epsg      ?logit_scaleFattention_biaslayer_norm_biasuse_qk_normN)__name__
__module____qualname__str__annotations__r   intr   r   r   r   r   floatr   r   r   r   boolr   r        ^/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/cohere.pyr   r      s    OKs"s"!!!!!J!J!NE!K ND !OT!Kr%   r   c                   $     e Zd Z fdZd Z xZS )LayerNorm2Dc                 h    t         |           t        j                  ||f      | _        || _        y N)super__init__mxzerosweighteps)selfd1d2r0   	__class__s       r&   r,   zLayerNorm2D.__init__    s)    hhBx(r%   c                 t    | j                   t        j                  j                  |d d | j                        z  S r*   )r/   r-   fast
layer_normr0   r1   xs     r&   __call__zLayerNorm2D.__call__%   s*    {{RWW//4txxHHHr%   r   r   r   r,   r:   __classcell__r4   s   @r&   r(   r(      s    
Ir%   r(   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                    t         |           || _        |j                  }|j                  x| _        }|j                  x| _        }|j                  |j                  z  }|dz  | _        |j                  }t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  ||z  ||      | _        |j                   | _        | j                   rNt#        | j
                  ||j$                        | _        t#        | j                  ||j$                        | _        t        j*                  |d|j,                        | _        y )Ng      ࿩bias)r0   T)traditionalbase)r+   r,   r@   r   r   n_headsr   
n_kv_headsscaler   nnLinearq_projk_projv_projo_projr   r(   r   q_normk_normRoPEr   rope)r1   r@   dimrF   rG   head_dimattetion_biasr4   s          r&   r,   zAttention.__init__*   s<   	!%!9!99w'+'?'??*##t'?'??t^
++iiWx%7mLiiZ(%:OiiZ(%:Oii( 2CmL++%dllH$BUBUVDK%t/B/BDK GGH$T__M	r%   r9   maskcachereturnc                    |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      }|j	                  ||| j                  d      }| j                  r"| j                  |      }| j                  |      }|j                  dddd      }|j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j!                  |
      S )Nr      r      )offset)rW   rH   rV   )shaperK   rL   rM   reshaperF   rG   r   rO   rP   	transposerR   r]   update_and_fetchr
   rH   rN   )r1   r9   rV   rW   BLDquerieskeysvaluesoutputs              r&   r:   zAttention.__call__E   s    ''1a $AAAv//!Qb9||Aq$//26kk'*G;;t$D##Aq!Q/~~aAq)1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r%   NNr   r   r   r   r,   r-   arrayr   r   r:   r<   r=   s   @r&   r?   r?   )   sY    NY N< $(#	!#88!# rxx !# }	!#
 
!#r%   r?   c                   $     e Zd Z fdZd Z xZS )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y )NFrB   )r+   r,   rI   rJ   	gate_projup_proj	down_proj)r1   rS   
hidden_dimr4   s      r&   r,   zMLP.__init__j   sJ    3
?yyju=:s?r%   c                 t    | j                  t        | j                  |      | j                  |                  S r*   )rq   r   ro   rp   r8   s     r&   r:   zMLP.__call__p   s)    ~~fT^^A%6QHIIr%   r;   r=   s   @r&   rm   rm   i   s    @Jr%   rm   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr@   c                 T   t         |           |j                  | _        |j                  | _        t        |      | _        t        |j                  |j                        | _	        t        j                  |j                  |j                  |j                        | _        || _        y )Nr0   rC   )r+   r,   r   r   rF   r?   	self_attnrm   r   mlprI   	LayerNormr   r   input_layernormr@   r1   r@   r4   s     r&   r,   zTransformerBlock.__init__u   s    ++//"4t'')?)?@!||$"5"5D<P<P 
 	r%   r9   rV   rW   rX   c                 |    | j                  |      }| j                  |||      }| j                  |      }||z   |z   S r*   )r{   rx   ry   )r1   r9   rV   rW   hattn_hff_hs          r&   r:   zTransformerBlock.__call__   sB       #4/xx{}q  r%   ri   rj   r=   s   @r&   ru   ru   t   sW    
Y 
 $(#		!88	! rxx 	! }		!
 
	!r%   ru   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )CohereModelr@   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        j                  |j                  |j                  |j                        | _        y c c}w )Nr   )r@   rw   )r+   r,   r@   r   r   rI   	Embeddingr   embed_tokensrangeru   layersrz   r   r   norm)r1   r@   _r4   s      r&   r,   zCohereModel.__init__   s    	//!%!7!7"""LL$:J:JK16t7M7M1N
1NA$'1N
 LL$"5"5D<P<P
	
s   C'inputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r   lenr   r	   zipr   )r1   r   rW   r~   rV   layercs          r&   r:   zCohereModel.__call__   su    
 f%=FS--E$Qa1DKK/HE1aq!A 0 yy|r%   r*   )	r   r   r   r   r,   r-   rk   r:   r<   r=   s   @r&   r   r      s%    
Y 
" r%   r   c                   X     e Zd Zdef fdZ	 ddej                  fdZed        Z	 xZ
S )Modelr@   c                 r    t         |           |j                  | _        t        |      | _        || _        y r*   )r+   r,   r   r   modelr@   r|   s     r&   r,   zModel.__init__   s,    // &
	r%   r   c                     | j                  ||      }| j                   j                  j                  |      }|| j                   j                  j                  z  }|S r*   )r   r   	as_linearr@   r   )r1   r   rW   outs       r&   r:   zModel.__call__   sK    
 jj'jj%%//4DJJOO///
r%   c                 .    | j                   j                  S r*   )r   r   )r1   s    r&   r   zModel.layers   s    zz   r%   r*   )r   r   r   r   r,   r-   rk   r:   propertyr   r<   r=   s   @r&   r   r      s9    Y   ! !r%   r   )dataclassesr   typingr   r   mlx.corecorer-   mlx.nnrI   activationsr   rE   r   r	   r
   r   Moduler(   r?   rm   ru   r   r   r$   r%   r&   <module>r      s    "      T T    I")) I=#		 =#@J")) J!ryy !2")) B!BII !r%   