
    (HJj+                     F   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 d dl
ZddlmZmZmZ e G d de             Z G d d	e	j"                        Z G d
 de	j"                        Z G d de	j"                        Z G d de	j"                        Z G d de	j"                        Zy)    )	dataclass)AnyOptionalN   )BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   d	Zeed
<   dZe	ed<   dZ
e	ed<   dZe	ed<   dZe	ed<   d Zy	)	ModelArgs
model_typen_embdn_layern_innern_headn_positionslayer_norm_epsilon
vocab_sizeNnum_key_value_headsTmulti_queryattention_biasmlp_biastie_word_embeddingsc                 \    | j                    | j                  rdn| j                  | _         y y )Nr   )r   r   r   selfs    c/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/gpt_bigcode.py__post_init__zModelArgs.__post_init__   s)    ##+,0,<,<q$++D$ ,    )__name__
__module____qualname__str__annotations__intfloatr   r   boolr   r   r   r    r   r   r   r      sf    OKLLKO##KNDHd $$Nr   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                    t         |           |j                  x| _        }|j                  x| _        }|j                  rdn|j                  x| _        }||z  x| _        }||z  | _	        |dz  | _
        t        |d      r|j                  }nd}t        j                  ||d| j                  z  z   |      | _        t        j                  |||      | _        y )Nr   g      r   F   bias)super__init__r   dimr   n_headsr   
n_kv_headshead_dimkv_dimscalehasattrr   nnLinearc_attnc_proj)r   r*   r1   r2   r3   r4   r   	__class__s          r   r0   zAttention.__init__#   s    $3!%,w,0,<,<q$++M*#&'>1 8+t^
4)*!00N"NiiS1t{{?%:PiiS~>r   xmaskcachereturnc                    |j                   \  }}}| j                  |      }t        j                  || j                  | j                  | j
                  z   gd      \  }}	}
|j                  ||| j                  d      j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|
j                  ||| j                  d      j                  dddd      }
||j                  |	|
      \  }	}
t        ||	|
|| j                  |      }|j                  dddd      j                  ||d      }| j                  |      S )N)axisr   r,   r      )r?   r6   r>   )shaper:   mxsplitr1   r5   reshaper2   	transposer3   update_and_fetchr	   r6   r;   )r   r=   r>   r?   BLDqkvquerieskeysvaluesoutputs               r   __call__zAttention.__call__8   s<    ''1akk!n "$((DHHt{{23"!
v
 //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aP 11$?LD&-T6djjt
 !!!Q1-55aB?{{6""r   NNr   r    r!   r   r0   rF   arrayr   r   rS   __classcell__r<   s   @r   r)   r)   "   sW    ?Y ?0 $(#	#88# rxx # }	#
 
#r   r)   c                   D     e Zd Zdef fdZdej                  fdZ xZS )MLPr*   c                     t         |           |j                  }|j                  }t	        |d      r|j
                  }nd}t        j                  |||      | _        t        j                  |||      | _	        y )Nr   Fr-   )
r/   r0   r   r   r7   r   r8   r9   c_fcr;   )r   r*   r1   
hidden_dimr   r<   s        r   r0   zMLP.__init__U   s`    kk\\
4$}}HHIIc:H=	ii
Ch?r   r@   c                 h    | j                  t        j                  | j                  |                  S N)r;   r8   gelur\   )r   r=   s     r   rS   zMLP.__call__b   s"    {{277499Q<011r   	r   r    r!   r   r0   rF   rV   rS   rW   rX   s   @r   rZ   rZ   T   s!    @Y @2RXX 2r   rZ   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr*   c                 t   t         |           |j                  | _        |j                  | _        t	        |      | _        t        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        || _        y )Neps)r/   r0   r   r   r)   attnrZ   mlpr8   	LayerNormr   ln_1ln_2r*   r   r*   r<   s     r   r0   zTransformerBlock.__init__g   sy    kkkkdO	t9LL$2I2IJ	LL$2I2IJ		r   r=   r>   r?   r@   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S r_   )rg   rj   rh   rk   )r   r=   r>   r?   rhouts          r   rS   zTransformerBlock.__call__q   sK     IIdiilD%0EHHTYYq\"!e
r   rT   rU   rX   s   @r   rc   rc   f   sW    Y  $(#	
88
 rxx 
 }	

 

r   rc   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )GPTBigCodeModelr*   c                    t         |           || _        |j                  | _        | j                  dkD  sJ t	        j
                  |j                  |j                        | _        t	        j
                  |j                  |j                        | _	        t        |j                        D cg c]  }t        |       c}| _        t	        j                  |j                  |j                        | _        y c c}w )Nr   )r*   re   )r/   r0   r*   r   r8   	Embeddingr   wter   wperanger   rc   ro   ri   r   ln_f)r   r*   _r<   s      r   r0   zGPTBigCodeModel.__init__   s    	//"""<<=<< 0 0$++>7<T\\7JK7J!"-7JKLL$2I2IJ	 Ls   .C:inputsc                    |j                   \  }}| j                  |      }|Bd gt        | j                        z  }t	        j
                  t        j                  |            }nFt	        j
                  t        j                  |d   j                  |d   j                  |z               }t        ||d         }|| j                  |      z  }t        | j                  |      D ]  \  }}	 ||||	      } | j                  |      S )Nr   )r?   )rE   ru   lenro   rF   rV   nparangeoffsetr   rv   ziprx   )
r   rz   r?   rK   rL   hidden_statesposition_idsr>   layercs
             r   rS   zGPTBigCodeModel.__call__   s    
 ||1(=FS[(E88BIIaL1L88BIIeAhoouQxQR?R$STL$]E!H=,//DFFE*HE1!-Q?M + yy''r   r_   ra   rX   s   @r   rr   rr   ~   s'    KY K ((r   rr   c                   X     e Zd Zdef fdZ	 ddej                  fdZed        Z	 xZ
S )Modelr*   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFr-   )r/   r0   r*   r   rr   transformerr   r8   r9   r   r   lm_headrl   s     r   r0   zModel.__init__   sW    	//*40''99T[[$//NDL (r   rz   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r_   )r   r*   r   ru   	as_linearr   )r   rz   r?   rp   s       r   rS   zModel.__call__   s[    
 vu-99((""&&005C 
 ,,s#C
r   c                 .    | j                   j                  S r_   )r   ro   r   s    r   layerszModel.layers   s    !!!r   r_   )r   r    r!   r   r0   rF   rV   rS   propertyr   rW   rX   s   @r   r   r      s;    OY O 

 " "r   r   )dataclassesr   typingr   r   mlx.corecorerF   mlx.nnr8   numpyr}   baser   r   r	   r   Moduler)   rZ   rc   rr   r   r'   r   r   <module>r      s    "      T T N N N(/#		 /#d2")) 2$ryy 0!(bii !(H"BII "r   