
    (HJjQ2                        d dl Z d dlmZ d dlmZmZmZmZ d dlm	Z
 d dlmZ ddlmZ ddlmZmZmZmZ ddlmZmZ ddlmZ e G d	 d
e             Z G d dej4                        Z G d dej4                        Ze
j:                  d        Z G d dej4                        Z G d dej4                        Z  G d dej4                        Z! G d dej4                        Z" G d dej4                        Z#y)    N)	dataclass)AnyListOptionalUnion   )swiglu)BaseModelArgscreate_attention_maskcreate_ssm_maskscaled_dot_product_attention)ArraysCacheKVCache)	SwitchGLUc                   $   e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZeeef   ed<   dZ	e
ed<   dZe
ed<   dZeee      ed<   dZe
ed<   d Zy)	ModelArgs
model_typehidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_headsattn_layer_offsetattn_layer_periodexpert_layer_offsetexpert_layer_periodmamba_d_convmamba_d_statemamba_expandnum_expertsnum_experts_per_tokrms_norm_epsmax_position_embeddings
vocab_sizeautomamba_dt_rankFmamba_proj_biasTmamba_conv_biasNlayers_block_typetie_word_embeddingsc                     | j                   dk(  r't        j                  | j                  dz        | _         | j                  Gt        | j                        D cg c]"  }|| j                  z  | j                  k(  rdnd$ c}| _        y y c c}w )Nr%      	attentionmamba)	r&   mathceilr   r)   ranger   r   r   )selfis     ]/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/jamba.py__post_init__zModelArgs.__post_init__/   s    '!%4+;+;b+@!AD!!) t556& 7A 4111T5K5KK   ! 7&D" *&s   'B)__name__
__module____qualname__str__annotations__intfloatr&   r   r'   boolr(   r)   r   r   r*   r5        r4   r   r      s    O  O%+M5c?+!OT! OT -1xS	*1 $$r?   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )JambaMLPargsc                 H   t         |           t        j                  |j                  |j
                  d      | _        t        j                  |j                  |j
                  d      | _        t        j                  |j
                  |j                  d      | _        y NFbias)	super__init__nnLinearr   r   	gate_projup_proj	down_projr2   rB   	__class__s     r4   rH   zJambaMLP.__init__>   sq    4#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXr?   xreturnc                 t    | j                  t        | j                  |      | j                  |                  S N)rM   r	   rK   rL   )r2   rP   s     r4   __call__zJambaMLP.__call__D   s)    ~~fT^^A%6QHIIr?   	r6   r7   r8   r   rH   mxarrayrT   __classcell__rO   s   @r4   rA   rA   =   s/    YY YJ"(( Jrxx Jr?   rA   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	JambaAttentionrB   c                    t         |           |j                  | _        |j                  | _        |j                  |j                  z  | _        | j
                  dz  | _        t        j                  |j                  |j                  | j
                  z  d      | _	        t        j                  |j                  |j                  | j
                  z  d      | _
        t        j                  |j                  |j                  | j
                  z  d      | _        t        j                  |j                  | j
                  z  |j                  d      | _        y )Ng      FrE   )rG   rH   r   r   r   head_dimscalerI   rJ   q_projk_projv_projo_projrN   s     r4   rH   zJambaAttention.__init__I   s   #'#;#; #'#;#; ((D,D,DD]]D(
iid66FU
 iid66FU
 iid66FU
 ii$$t}}4d6F6FU
r?   rP   maskcacherQ   c                 x   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	||j                  ||	      \  }}	t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r      )rd   r^   rc   )shaper_   r`   ra   reshaper   	transposer   update_and_fetchr   r^   rb   )r2   rP   rc   rd   BLDquerieskeysvaluesoutputs              r4   rT   zJambaAttention.__call__]   s2    ''1a $AAAv//!Q(@(@"EOOq!Q
 ||Aq$":":B?II!QPQSTU1d&>&>CMMq!Q
  11$?LD&-T6djjt
 !!!Q1-55aB?{{6""r?   NNr6   r7   r8   r   rH   rV   rW   r   r   rT   rX   rY   s   @r4   r[   r[   H   sW    
Y 
. $(#	#88# rxx # }	#
 
#r?   r[   c                     | |z  |z   S rS   r>   )abcs      r4   fmarz   z   s    q519r?   c                   8     e Zd Zdef fdZddZd Zd Z xZS )JambaMambaMixerrB   c                    t         |           |j                  | _        |j                  | _        |j
                  | _        |j                  |j                  z  | _        |j                  | _
        |j                  | _        |j                  | _        t        j                   | j                  | j                  dz  | j                        | _        t        j$                  | j                  | j                  | j                  | j                  | j                  d      | _        t        j                   | j                  | j                  | j                  dz  z   d      | _        t        j                   | j                  | j                  d      | _        t-        j.                  t-        j0                  d| j                  dz         j3                  d| j                  g      | j                  d	      }t-        j4                  |      | _        t-        j8                  | j                  g      | _        t        j                   | j                  | j                  | j                        | _        t        j>                  | j                  |j@                  
      | _!        t        j>                  | j                  |j@                  
      | _"        t        j>                  | j                  |j@                  
      | _#        y )Nrg   rE   r   )in_channelsout_channelskernel_sizegroupsrF   paddingFTg      ?r   )repeatsaxiseps)$rG   rH   r   r   ssm_state_sizer   conv_kernel_sizer   r   r&   time_step_rankr(   use_conv_biasr'   use_biasrI   rJ   in_projConv1dconv1dx_projdt_projrV   repeatarangerj   logA_logonesro   out_projRMSNormr"   dt_layernormb_layernormc_layernorm)r2   rB   ArO   s      r4   rH   zJambaMambaMixer.__init__   s;   ++"00 $ 1 1!%!2!2T5E5E!E"00!11,,yyd44q8t}}
 ii..//--))##
 ii""$"5"5"99

 yy!4!4d6L6LSWXIIIIc4..45==q$BUBU>VW**

 VVAY
$0012		""D$4$44==
 JJt':':@Q@QR::d&9&9t?P?PQ::d&9&9t?P?PQr?   c                 .   |j                   d   }| j                  }| j                  |      }t        j                  || j
                  | j
                  | j                  z   gd      \  }}}	| j                  |      | j                  |      | j                  |	      }	}}t        j                  | j                  |            }t        j                  ||z  d      t        j                  |d      z  }
t        j                  t        j                  |d      |z        }t        |      D ]/  }|"t!        ||d d |f   |
d d |f         |
d d |f<   |
d d |f   }1 |
t        j                  |	d      z  j#                  d      }|||z  z   }||
d d df   fS )Nr   rf   r   )ri   ro   r   rV   splitr   r   r   r   r   rI   softplusr   expand_dimsexpr1   rz   squeeze)r2   rP   r   stateTro   deltaBCdeltarm   C	new_statedtAtys                 r4   ssm_stepzJambaMambaMixer.ssm_step   st   GGAJFF++a.hh  $"5"58K8K"KL
q!
 ''.0@0@0CTEUEUVWEX!qDLL/0NN519b1BNN1b4II	ffR^^E2.23 qA "%eSAY	!Q$"H	!Q$adOE  2..77;AI)ArE"""r?   c                    | j                  |      }|j                  dd      \  }}| j                  }|t        j                  ||gd      }nt        j
                  |d|dz
  dfdg      }| j                  |      }|d d |dz
   d d d f   }t        j                  |      }t        j                  | j                         }	| j                  ||	|      \  }
}| j                  t        ||
            }|||ffS )Nrg   rf   )indices_or_sectionsr   r   r   )r   r   r   )r   r   r   rV   concatenatepadr   rI   silur   r   r   r   r	   )r2   rP   
conv_state	ssm_statexzzKx_fullconv_outr   r   s              r4   _process_sequencez!JambaMambaMixer._process_sequence   s    \\!_xxABx71!!!^^ZO!<FVVAQ
F;<F;;v&AQxz1,-
GGHVVDJJ}}Q959MM&A,':y)))r?   c                 t    |d\  }}n
|d   |d   }}| j                  |||      \  }\  }}|
||d<   ||d<   |S )Nrt   r   r   )r   )r2   rP   rd   r   r   rs   s         r4   rT   zJambaMambaMixer.__call__   sb    =$.!J	$)!HeAh	J*.*@*@z9+
''Y !E!H E!Hr?   rS   )	r6   r7   r8   r   rH   r   r   rT   rX   rY   s   @r4   r|   r|      s!    +RY +RZ#.* r?   r|   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )JambaSparseMoeBlockrB   c                    t         |           |j                  | _        t        j                  |j
                  |j                  d      | _        t        |j
                  |j                  |j                        | _
        y rD   )rG   rH   r!   rI   rJ   r   r    routerr   r   
switch_mlprN   s     r4   rH   zJambaSparseMoeBlock.__init__   sa    #'#;#; ii 0 0$2B2BO#d44d6F6F
r?   rP   rQ   c                 `   | j                  |      }| j                  }t        j                  t        j                  | |dz
  d      dd |f         }t        j
                  ||d      }t        j                  |dd      }| j                  ||      }||d   z  j                  d	      }|S )
Nr   rf   )kthr   .r   T)r   precise).Nr   )	r   r!   rV   stop_gradientargpartitiontake_along_axissoftmaxr   sum)r2   rP   gateskindsscoresr   s          r4   rT   zJambaSparseMoeBlock.__call__   s    A$$AE KCQSRSQSG TU##E4b9FT:OOAt$	""''R'0r?   rU   rY   s   @r4   r   r      s*    
Y 
"(( rxx r?   r   c            	            e Zd Zdededef fdZ	 	 d
dej                  de	ej                     de	e
   dej                  fd	Z xZS )JambaDecoderLayerrB   
layer_type	layer_idxc                    t         |           |dk(  | _        | j                  rt        |      | _        nt        |      | _        |j                  dkD  r&||j                  z   |j                  z  dk(  rt        }nt        } ||      | _        t        j                  |j                  |j                         | _        t        j                  |j                  |j                         | _        y )Nr-   r   r   r   )rG   rH   is_attnr[   	self_attnr|   r.   r    r   r   r   rA   feed_forwardrI   r   r   r"   input_layernormpre_ff_layernorm)r2   rB   r   r   ffn_layer_classrO   s        r4   rH   zJambaDecoderLayer.__init__   s    ![0<<+D1DN(.DJq T5559Q9QQUVV1O&O+D1!zz$*:*:@Q@QR "

4+;+;ARAR Sr?   rP   rc   rd   rQ   c                     | j                   r#| j                  | j                  |      ||      }n!| j                  | j                  |      |      }||z   }|| j	                  | j                  |            z   }|S rS   )r   r   r   r.   r   r   )r2   rP   rc   rd   hrouts          r4   rT   zJambaDecoderLayer.__call__  sq     <<t33A6eDA

4//2E:AE$##D$9$9!$<==
r?   rt   )r6   r7   r8   r   r9   r;   rH   rV   rW   r   r   rT   rX   rY   s   @r4   r   r      si    TY TC TC T* $(#	88 rxx  }	
 
r?   r   c                   j     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	 xZ
S )
JambaModelrB   c           	         t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  \  }}t        |||       c}}| _
        t        j                  |j
                  |j                        | _        |j                  j                  d      | _        |j                  j                  d      | _        y c c}}w )Nr   r-   r.   )rG   rH   rI   	Embeddingr$   r   embed_tokens	enumerater)   r   layersr   r"   final_layernormindexattn_idxssm_idx)r2   rB   idxr   rO   s       r4   rH   zJambaModel.__init__  s    LL$:J:JK $D$:$:;
;Q dAs+;
  "zz$*:*:@Q@QR..44[A--33G<
s   C(inputsrd   rQ   c                 R   | j                  |      }|d gt        | j                        z  }t        ||| j                           }t        ||| j                           }t        | j                  |      D ]   \  }}|j                  r|n|} ||||      }" | j                  |      S )N)rc   rd   )
r   lenr   r   r   r   r   zipr   r   )	r2   r   rd   r   	attn_maskssm_masklayerry   rc   s	            r4   rT   zJambaModel.__call__)  s    
 f%=FS--E)!U4==-AB	"1eDLL&9:DKK/HE1 %98Dad!,A 0 ##A&&r?   rS   ru   rY   s   @r4   r   r     sA    
=Y 
=  $'' }' 
	'r?   r   c                        e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	d Z
d Zed	        Zed
        Z xZS )ModelrB   c                     t         |           |j                  | _        || _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y rD   )rG   rH   r   rB   r   modelr*   rI   rJ   r   r$   lm_headrN   s     r4   rH   zModel.__init__>  sX    //	%
''99T%5%5tUSDL (r?   r   rd   rQ   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S rS   )r   rB   r*   r   	as_linearr   )r2   r   rd   r   s       r4   rT   zModel.__call__F  sW    
 jj'99((**))33C8C 
 ,,s#C
r?   c                     g }| j                   j                  D ]C  }|j                  r|j                  t	                      )|j                  t        d             E |S )Nrg   )size)r   r   r   appendr   r   )r2   cachesr   s      r4   
make_cachezModel.make_cacheR  sH    ZZ&&E}}gi(kq12	 '
 r?   c                   	 t        |j                               D ]2  \  }}d|v s|j                  d   dk7  s|j                  dd      ||<   4 | j                  j
                  r|j                  dd        t        | j                  j                        D ]  }d| d	t        	fd|j                         D              s,d	D ]v  }d
D ]o  }t        t        |            D cg c]-  }	 d| d| d| |v r|j                  	 d| d| d|       / }}|sPt        j                  |      |	 d| d| <   q x  |S c c}w )Nzconv1d.weightrf   r   rg   zlm_head.weightzmodel.layers.z.feed_forwardc              3   F   K   | ]  }|j                   d         yw)	.experts.N)
startswith).0keybases     r4   	<genexpr>z!Model.sanitize.<locals>.<genexpr>e  s"     T^cs~~i&89^s   !)rK   rM   rL   )weightrF   scalesbiasesr   .z.switch_mlp.)listitemsri   moveaxisrB   r*   popr1   r   anyrq   r   rV   stack)
r2   weightsr   vlprojnameeexpert_tensorsr  s
            @r4   sanitizezModel.sanitize[  s[   )DAq!#q(8ZZ1-
 * 99((KK($/tyy223A"1#]3DTW\\^TT=BD "'s7|!4&!4A"V9QCqav>'I  tfIaS$q$GH!4 # &
 &FHhh*G4&TF!D6 BC C > 4" &s   #2E
c                 .    | j                   j                  S rS   )r   r   )r2   s    r4   r   zModel.layersv  s    zz   r?   c                     d }|S )Nc                 0    | j                  d      rdddS y)Nr   @      )
group_sizebitsT)endswith)path_s     r4   	predicatez(Model.quant_predicate.<locals>.predicate|  s    }}X&&(!44r?   r>   )r2   r  s     r4   quant_predicatezModel.quant_predicatez  s    	
 r?   rS   )r6   r7   r8   r   rH   rV   rW   r   r   rT   r   r  propertyr   r   rX   rY   s   @r4   r   r   =  su    TY T  $

 }
 
	
6 ! !  r?   r   )$r/   dataclassesr   typingr   r   r   r   mlx.corecorerV   mlx.nnrI   activationsr	   r  r
   r   r   r   rd   r   r   switch_layersr   r   ModulerA   r[   compilerz   r|   r   r   r   r   r>   r?   r4   <module>r+     s     ! - -     ( $ $ $ $NJryy J/#RYY /#d  cbii cL")) *		 D' 'BDBII Dr?   