
    (HJj$                        d dl mZ d dlmZmZ d dlmZ d dlm	Z	 d dl
ZddlmZ ddlmZmZmZ e G d de             Z G d	 d
e	j&                        Z G d de	j&                        Z G d de	j&                        Z G d de	j&                        Z G d de	j&                        Z G d de	j&                        Z G d de	j&                        Z G d de	j&                        Zy)    )	dataclass)AnyOptionalN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                   T    e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   y)		ModelArgs
model_type
vocab_sized_model
ffn_configattn_confign_layersn_headsN)__name__
__module____qualname__str__annotations__intdict     \/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/dbrx.pyr   r      s'    OOLMLr   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                    t         |           |j                  | _        |j                  | _        |j                  |j                  z  | _        |j                  d   | _        |j                  d   | _        |j                  d   | _	        | j
                  dz  | _
        t        j                  |j                  | j                  dz  | j                  z   | j
                  z  d      | _        t        j                  |j                  |j                  d      | _        t        j                  | j
                  d| j                        | _        y )	N
kv_n_headsclip_qkv
rope_thetag         Fbias)traditionalbase)super__init__r   	num_headsr   head_dimr   num_key_value_headsr#   r$   scalennLinearWqkvout_projRoPEropeselfr    	__class__s     r   r+   zAttention.__init__   s    ||4#'#3#3L#A ((4**<8]]D(
IILL%%)DNN:dmmK
	
 		$,,5IGGMM
	r   xmaskcachereturnc                    | j                  |      }t        j                  || j                   | j                        }| j                  | j                  | j
                  | j                  z  z   g}t        j                  ||d      \  }}}|j                  \  }	}
}|j                  |	|
| j                  d      j                  dddd      }|j                  |	|
| j                  d      j                  dddd      }|j                  |	|
| j                  d      j                  dddd      }|P| j                  ||j                        }| j                  ||j                        }|j                  ||      \  }}n"| j                  |      }| j                  |      }t        ||||| j                   |	      }|j                  dddd      j                  |	|
d      }| j#                  |      S )
N)a_mina_maxaxisr   r%   r      )offset)r;   r/   r:   )r2   mxclipr#   r   r-   r.   splitshapereshaper,   	transposer5   rD   update_and_fetchr
   r/   r3   )r7   r9   r:   r;   qkvsplitsquerieskeysvaluesBLDoutputs                r   __call__zAttention.__call__1   s    iilggc$--t}}E,,t}}t?W?W/W WX "f2 >v''1a //!Q;EEaAqQ||Aq$":":B?II!QPQSTU1d&>&>CMMq!Q
 iii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?}}V$$r   NNr   r   r   r   r+   rE   arrayr   r   rU   __classcell__r8   s   @r   r   r      sW    
Y 
4 $(#	!%88!% rxx !% }	!%
 
!%r   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	NormAttnNormr    c                     t         |           t        j                  |j                  d      | _        t        j                  |j                  d      | _        t        |      | _        y NFr&   )	r*   r+   r0   	LayerNormr   norm_1norm_2r   attnr6   s     r   r+   zNormAttnNorm.__init__V   sE    ll4<<e<ll4<<e<dO	r   r9   r:   r;   r<   c                 x    | j                  | j                  |      ||      }||z   }|| j                  |      fS )N)r:   r;   )rb   r`   ra   )r7   r9   r:   r;   hs        r   rU   zNormAttnNorm.__call__\   s<     IIdkk!n4uI=E$++a.  r   rV   rW   rZ   s   @r   r\   r\   U   sW    $Y $ $(#	!88! rxx ! }	!
 
!r   r\   c                   `     e Zd Zdedef fdZdej                  dej                  fdZ xZS )MLPr   ffn_dimc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y r^   )r*   r+   r0   r1   v1w1w2)r7   r   rg   r8   s      r   r+   zMLP.__init__h   sJ    ))GW59))GW59))GW59r   r9   r<   c                 |    t        | j                  |      | j                  |            }| j                  |      }|S N)r   rj   ri   rk   )r7   r9   current_hidden_statess      r   rU   zMLP.__call__n   s5     &twwqz4771: > $(= >$$r   	r   r   r   r   r+   rE   rX   rU   rY   rZ   s   @r   rf   rf   g   s1    : :c :%"(( %rxx %r   rf   c                   H     e Zd Zdedef fdZdej                  fdZ xZS )Routerr   num_expertsc                 \    t         |           t        j                  ||d      | _        y r^   )r*   r+   r0   r1   layer)r7   r   rr   r8   s      r   r+   zRouter.__init__u   s"    YYw%@
r   r9   c                 $    | j                  |      S rm   )rt   )r7   r9   s     r   rU   zRouter.__call__y   s    zz!}r   ro   rZ   s   @r   rq   rq   t   s)    A A# A"(( r   rq   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )SparseMoeBlockr    c                    t         |           |j                  | _        |j                  d   | _        |j                  d   | _        |j                  d   | _        t        | j                  | j
                        | _        t        | j
                        D cg c]"  }t        | j                  | j                        $ c}| _        y c c}w )Nffn_hidden_sizemoe_num_experts	moe_top_k)r*   r+   r   r   rg   rr   num_experts_per_tokrq   routerrangerf   expertsr7   r    _r8   s      r   r+   zSparseMoeBlock.__init__~   s    ||'89??+<=#'??;#? T\\4+;+;<5:4;K;K5L
5LCdll+5L
 
s   'C	r9   r<   c           	      Z   | j                   }|j                  }|j                  d|j                  d         }| j                  |      }t	        j
                  |j                  t        j                        d      }t	        j                  t	        j                  | |dz
  d      d d d |f         }t	        j                  ||d      }|t        j                  j                  |ddd      z  }|j                  |j                        }| j                  rt        j                   |      }t	        j"                  |j                  d   ||j                  d   f|j                        }t%        | j&                        D ]X  \  }}	t)        t        j                   t        j*                  ||k(              \  }
}|
j,                  dk(  rI |	||
         ||
|f<   Z ||d d d d d f   z  j/                  d      }ng }t1        |||j3                               D ]d  \  }}}t	        j4                  |D cg c]  } | j&                  |   |       c}d      }||z  j/                  d      }|j7                  |       f t	        j4                  |d      }|j                  |      S c c}w )Nr@   rA   r   )kthrB   T)ordrB   keepdimsr   )r|   rH   rI   r}   rE   softmaxastypefloat32stop_gradientargpartitiontake_along_axislinalgnormdtypetrainingnprX   zeros	enumerater   mapwheresizesumziptoliststackappend)r7   r9   ne
orig_shapegatesindsscoresyeexpertidx1idx2xtstityts                   r   rU   zSparseMoeBlock.__call__   s'   %%WW
IIb!''"+&A

5<<

3"=BF LQPSQSPSV TU##E4b9"))..QR$.OOqww'==88D>D!''!*b!''"+6@A&t||4	6 288DAI+>?
d99> &qw$*	 5 VAq$J'',,!,4AA!!VT[[];
BXXB?Bqt||Ar2B?bI2g]]]+ < #Ayy$$ @s   3J(
	r   r   r   r   r+   rE   rX   rU   rY   rZ   s   @r   rw   rw   }   s*    

Y 

%"(( %rxx %r   rw   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	DecoderLayerr    c                 b    t         |           t        |      | _        t	        |      | _        y rm   )r*   r+   rw   ffnr\   norm_attn_normr6   s     r   r+   zDecoderLayer.__init__   s'    !$'*40r   r9   r:   r;   r<   c                 Z    | j                  |||      \  }}| j                  |      |z   }|S rm   )r   r   )r7   r9   r:   r;   rrd   outs          r   rU   zDecoderLayer.__call__   s2     ""1dE21hhqkAo
r   rV   rW   rZ   s   @r   r   r      sW    1Y 1 $(#	88 rxx  }	
 
r   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )DBRXr    c                 \   t         |           |j                  | _        t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j
                  d      | _        y c c}w )N)r    Fr&   )r*   r+   r   r0   	Embeddingr   wter~   r   r   blocksr_   norm_fr   s      r   r+   zDBRX.__init__   su    //<<>8=dmm8LM8L1|.8LMll4<<e< Ns   'B)inputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r   lenr   r	   r   r   )r7   r   r;   rd   r:   rt   cs          r   rU   zDBRX.__call__   ss    
 HHV=FS--E$Qa1DKK/HE1aq!A 0 {{1~r   rm   r   rZ   s   @r   r   r      s%    =Y = r   r   c                   ^     e Zd Zdef fdZ	 ddej                  fdZed        Z	d Z
 xZS )Modelr    c                     t         |           |j                  | _        t        |      | _        t        j                  |j                  |j                  d      | _	        || _
        y r^   )r*   r+   r   r   transformerr0   r1   r   r   lm_headr    r6   s     r   r+   zModel.__init__   sH    //:yytUK	r   r   c                 H    | j                  ||      }| j                  |      S rm   )r   r   )r7   r   r;   r   s       r   rU   zModel.__call__   s%    
 vu-||C  r   c                 .    | j                   j                  S rm   )r   r   )r7   s    r   layerszModel.layers   s    &&&r   c           
      6   | j                   j                  d   }| j                   j                  d   }d}|j                         D ci c]  \  }}||vs|| }}}|j                         D ]  \  }}||v st        t	        j
                  ||d            D 	cg c]  \  }}	|j                  dd|       dz   |	f! }
}}	|j                  d	      r |
D 	cg c]  \  }}	||	j                  f }
}}	|j                  |
        |S c c}}w c c}	}w c c}	}w )
Nrz   ry   zexperts.mlpr   rA   z.mlp.z.weightrk   )
r    r   itemsr   rE   rG   replaceendswithTupdate)r7   weightsrr   dimpatternkvnew_weightsr   svr   ss               r   sanitizezModel.sanitize   s   ii**+<=ii""#45(/L17!;Kq!tLMMODAq!| "+288A{+K!L!L2 YYv1#w/);R@!L   ::d#6=>gUQ244ygG>""7+ $  M
 ?s   D	D	$DDrm   )r   r   r   r   r+   rE   rX   rU   propertyr   r   rY   rZ   s   @r   r   r      s>    Y  !! ' 'r   r   )dataclassesr   typingr   r   mlx.corecorerE   mlx.nnr0   numpyr   activationsr   r)   r   r	   r
   r   Moduler   r\   rf   rq   rw   r   r   r   r   r   r   <module>r      s    "       T T   9%		 9%x!299 !$
%")) 
%RYY ,%RYY ,%^299 "299 4$BII $r   