
    (HJjw!                     j   d dl Z d dlmZ d dlmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZ ddlmZ ddlmZ e G d	 d
e             Z G d dej,                        Z G d dej,                        Z G d dej,                        Z G d dej,                        Z G d dej,                        Zy)    N)	dataclass)OptionalTupleUnion   )swiglu)BaseModelArgscreate_ssm_mask)ArraysCache)
ssm_updatec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eeef   ed<   e	eef   ed<   dZ
ee   ed<   dZeed<   d Zy)	ModelArgs
model_type	num_headshead_dim
vocab_sizehidden_sizeintermediate_size
state_sizenum_hidden_layerslayer_norm_epsilonconv_kerneln_groupsuse_biasuse_conv_biastie_word_embeddingstime_step_limittime_step_rankNssm_state_sizei  max_position_embeddingsc                     | j                   dk(  r't        j                  | j                  dz        | _         | j                  | j
                  | _        y y )Nauto   )r   mathceilr   r   r   selfs    ^/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/mamba2.py__post_init__zModelArgs.__post_init__%   sI    &("&))D,<,<r,A"BD&"&//D '    )__name__
__module____qualname__str__annotations__intfloatboolr   r   r   r   r    r)    r*   r(   r   r      s    ONMOOMN5%<((#s(O#$(NHSM(#'S'2r*   r   c                   |     e Zd Zddedef fdZd	dej                  dej                  dej                  fdZ xZ	S )
MambaRMSNormGatedr   epsc                 d    t         |           || _        t        j                  |      | _        y N)super__init__r6   mxonesweight)r'   r   r6   	__class__s      r(   r:   zMambaRMSNormGated.__init__-   s%    ggk*r*   hidden_statesgatereturnc                     |t        ||      }t        j                  j                  || j                  | j
                        S r8   )r   r;   fastrms_normr=   r6   )r'   r?   r@   s      r(   __call__zMambaRMSNormGated.__call__2   s5    "47Mwwt{{DHHEEr*   )gư>r8   )
r+   r,   r-   r0   r1   r:   r;   arrayrE   __classcell__r>   s   @r(   r5   r5   ,   s@    +C +e +
Fbhh Fbhh F"(( Fr*   r5   c                       e Zd Zdedef fdZdej                  dee	   deej                     dej                  fdZ
d	ej                  d
ej                  dej                  dej                  dee	   deej                     dej                  fdZ	 dd	ej                  deej                     dee	   dej                  fdZ xZS )Mamba2Blockargs	layer_idxc                 p   t         |           || _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  |j                  z  | _	        |j                  | _
        |j                  | _        |j                  | _        |j                  | _        | j                  | j                  z  | _        |j                  | _        | j                  d| j                  z  | j
                  z  z   | _        t!        j"                  | j                  | j                  |j                  d| j                  |j                        | _        | j                  | j                  z   | j                  z   }t!        j&                  | j                  ||j                        | _        t+        j,                  | j                        | _        t+        j0                  t+        j2                  d| j                  dz   t*        j4                              | _        t+        j,                  | j                        | _        t;        | j                  |j<                        | _        t!        j&                  | j                  | j                  |j                        | _         y )N   r   )in_channelsout_channelskernel_sizepaddinggroupsbiasrT   r   dtyper6   )!r9   r:   rL   r   r   r   r   conv_kernel_sizer   r   r   r   r   heads_per_groupr   conv_dimnnConv1dconv1dLinearin_projr;   r<   dt_biaslogarangefloat32A_logDr5   r   normout_proj)r'   rK   rL   projection_sizer>   s       r(   r:   zMamba2Block.__init__9   s   "++"11 $ 0 0!%$--!?!//#33#~~>..T]]1BTEXEX1XXii((==##
 004==@4>>Qyy!1!1?Wwwt~~.VVBIIa!);2::NO
(%""(?(?
	 		""D$4$44==
r*   
conv_inputcachemaskrA   c                    |t        j                  |d   |d      }||d   It        j                  |j                  d   | j                  dz
  | j
                  f|j                        }n|d   }t        j                  ||gd      }| j                  dz
  }|j                  q|j                  d   }t        j                  |j                  d||z
        }|d d d f   t        j                  |      z   d   }	t        j                  ||	d      |d<   n;|d d | d d d f   |d<   n(t        j                  |d| j                  dz
  dfdg      }| j                  |      }
t        j                  |
      S )N).Nr   r   rV   axis)r   r   )r;   wherezerosshaperY   r[   rW   concatenatelengthscliprc   take_along_axispadr^   r\   silu)r'   rj   rk   rl   
conv_statepadded_inputn_keeptends	positionsconv_outputs              r(   _convzMamba2Block._conva   sh    $y/:qAJQxXX%%a($*?*?!*CT]]S$**

 #1X
>>:z*BKL**Q.F}}( &&q)wwu}}aV<!!T']RYYv->>	J	--lIANa'F78Q7a66Vd&;&;a&?%CVLL kk,/ww{##r*   r?   BCdtc                    |j                   \  }}}	|j                  ||| j                  | j                        }|j                  ||| j                  | j
                        }|j                  ||| j                  | j
                        }|r|d   }
|j                  }nd\  }
}t        || j                  ||| j                  || j                  |
| j                  ||      \  }}
|r|
|d<   |j                  ||| j                        S )Nr   )NN)rr   reshaper   r   r   r   rt   r   re   rf   ra   r   r   )r'   r?   r   r   r   rk   rl   
batch_sizeseq_len_statert   ys                r(   _ssmzMamba2Block._ssm   s     "/!4!4
GQ%--
 IIj'4==$:M:MNIIj'4==$:M:MN!HEmmG'NE7JJFFLL  
5 E!HyyWd.D.DEEr*   c                    | j                  |      }t        j                  || j                  | j                  | j                  z   gd      \  }}}| j                  |||      }t        j                  || j                  | j                  | j                  | j                  z  z   gd      \  }}	}
| j                  ||	|
|||      }|r|j                  |j                  d          | j                  ||      }| j                  |      S )Nrn   )rl   r   )r`   r;   splitr   r[   r   r   r   r   advancerr   rg   rh   )r'   r?   rl   rk   	projectedr@   rj   r   r   r   r   r   s               r(   rE   zMamba2Block.__call__   s     LL/	!xx##T%;%;dmm%KL 
j"
 jjUD9 hh&&&&9L9L)LL 
q! IImQ2u4I@MM!''!*%IIa}}Qr*   r8   )r+   r,   r-   r   r0   r:   r;   rF   r   r   r   r   rE   rG   rH   s   @r(   rJ   rJ   8   s   &
Y &
3 &
P $HH $ $ $ rxx 	 $
 
 $D#Fxx#F 88#F 88	#F
 HH#F $#F rxx #F 
#FR (,	 xx  rxx   $	 
 
 r*   rJ   c            	            e Zd Zdedef fdZ	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
ResidualBlockrK   rL   c                     t         |           t        ||      | _        t	        j
                  |j                        | _        y r8   )r9   r:   rJ   mixerr\   RMSNormr   rg   )r'   rK   rL   r>   s      r(   r:   zResidualBlock.__init__   s2     y1
JJt//0	r*   xrl   rk   rA   c                 P    | j                  | j                  |      ||      }||z   S r8   )r   rg   )r'   r   rl   rk   outputs        r(   rE   zResidualBlock.__call__   s(     DIIaL$6zr*   r8   )r+   r,   r-   r   r0   r:   r;   rF   r   r   rE   rG   rH   s   @r(   r   r      sX    1Y 13 1 UY!)"((!3<D[<Q	r*   r   c                   p     e Zd Zdef fdZ	 ddej                  deee	      dej                  fdZ
 xZS )Mamba2rK   c                 \   t         |           || _        t        j                  |j
                  |j                        | _        t        |j                        D cg c]  }t        ||       c}| _        t        j                  |j                  |j                        | _        y c c}w )NrX   )r9   r:   rK   r\   	Embeddingr   r   
embeddingsranger   r   layersr   r   norm_f)r'   rK   ir>   s      r(   r:   zMamba2.__init__   s    	,,t8H8HI7<T=S=S7TU7T!}T1-7TUjj!1!1t7N7NO Vs   B)r   rk   rA   c                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r   lenr   r
   zipr   )r'   r   rk   hiddenrl   layercs          r(   rE   zMamba2.__call__   st     #=FS--EvuQx0DKK/HE164+F 0 {{6""r*   r8   )r+   r,   r-   r   r:   r;   rF   r   listr   rE   rG   rH   s   @r(   r   r      sG    PY P AE##"*4+<"=#	#r*   r   c                        e Zd Zdef fdZ	 ddej                  deee	      dej                  fdZ
ddedee	   fdZed	        Zd
 Z xZS )ModelrK   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFrU   )r9   r:   rK   r   r   backboner   r\   r_   r   r   lm_head)r'   rK   r>   s     r(   r:   zModel.__init__   sW    	//t''99T%5%5tUSDL (r*   inputsrk   rA   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r8   )r   rK   r   r   	as_linearr   )r'   r   rk   r   logitss        r(   rE   zModel.__call__   sW     vu-99((]]--77?F  \\&)Fr*   r   c                 z    t        | j                  j                        D cg c]  }t        d       c}S c c}w )NrN   )size)r   rK   r   r   )r'   r   r   s      r(   
make_cachezModel.make_cache   s0    -24993N3N-OP-O#-OPPPs   8c                 .    | j                   j                  S r8   )r   r   r&   s    r(   r   zModel.layers   s    }}###r*   c                     |j                         D ]2  \  }}d|v s|j                  d   dk7  s|j                  dd      ||<   4 |S )Nzconv1d.weightr   r   rN   )itemsrr   moveaxis)r'   weightskvs       r(   sanitizezModel.sanitize  sH    MMODAq!#q(8ZZ1-
 $ r*   r8   )r   )r+   r,   r-   r   r:   r;   rF   r   r   r   rE   r0   r   propertyr   r   rG   rH   s   @r(   r   r      s{    TY T FJ	hh	'/[0A'B			QS Qk1B Q $ $r*   r   )r$   dataclassesr   typingr   r   r   mlx.corecorer;   mlx.nnr\   activationsr   baser	   r
   rk   r   ssmr   r   Moduler5   rJ   r   r   r   r3   r*   r(   <module>r      s     ! ) )    0   2 2 26	F		 	FI ")) I X
BII 
#RYY #. BII  r*   