
    GJj5                     X   d dl Z d dlmZ d dlmZmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZmZmZmZmZ e G d d             Z G d d	e
j(                        Z G d
 de
j(                        Z G d de
j(                        Z G d de
j(                        Z G d de
j(                        Zy)    N)	dataclass)LiteralOptional)tree_flatten)LocalRelPositionalEncodingMultiHeadAttentionRelPositionalEncodingRelPositionMultiHeadAttention"RelPositionMultiHeadLocalAttentionc                      e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   dZeed<   dZeed<   dZ	eed<   dZ
eej                     ed<   dZeej                     ed<   dZeed<   dZeee      ed<   y)ConformerArgsfeat_inn_layersd_modeln_headsff_expansion_factorsubsampling_factorself_attention_modelsubsamplingconv_kernel_sizesubsampling_conv_channelspos_emb_max_lenFcausal_downsamplingTuse_biasxscalingN
pos_bias_u
pos_bias_v    subsampling_conv_chunking_factoratt_context_size)__name__
__module____qualname__int__annotations__strr   boolr   r   r   r   mxarrayr   r   r    list     `/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/parakeet_mlx/conformer.pyr   r      s    LMLL"" %%HdHd%)J")%)J"),-$c-,0htCy)0r,   r   c                   f     e Zd Zddededef fdZdej                  dej                  fdZ xZ	S )	FeedForwardr   d_ffr   c                     t         |           t        j                  |||      | _        t        j
                         | _        t        j                  |||      | _        y )Nbias)super__init__nnLinearlinear1SiLU
activationlinear2)selfr   r0   r   	__class__s       r-   r5   zFeedForward.__init__)   sC    yy$X>'')yywX>r,   xreturnc                 `    | j                  | j                  | j                  |                  S N)r;   r:   r8   )r<   r>   s     r-   __call__zFeedForward.__call__/   s"    ||DOODLLO<==r,   )T)
r!   r"   r#   r$   r'   r5   r(   r)   rB   __classcell__r=   s   @r-   r/   r/   (   s8    ? ?3 ?$ ?>"(( >rxx >r,   r/   c                   ^     e Zd Zdef fdZddej                  dej                  fdZ xZS )Convolutionargsc           	         |j                   dz
  dz  dk(  sJ t        | 	          |j                   dz
  dz  | _        t	        j
                  |j                  |j                  dz  ddd|j                        | _        t	        j
                  |j                  |j                  |j                   dd|j                  |j                        | _	        t	        j                  |j                        | _        t	        j                         | _        t	        j
                  |j                  |j                  ddd|j                        | _        y )Nr      r   )kernel_sizestridepaddingr3   )rJ   rK   rL   groupsr3   )r   r4   r5   rL   r6   Conv1dr   r   pointwise_conv1depthwise_conv	BatchNorm
batch_normr9   r:   pointwise_conv2)r<   rG   r=   s     r-   r5   zConvolution.__init__4   s   %%)Q.!333--1a7!yyLLLL1 
 !iiLLLL--<<
 ,,t||4'')!yyLLLL 
r,   r>   r?   c                 |   | j                  |      }t        j                  |d      }||j                  || j                        }n/t        j                  |d| j                  | j                  fdf      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }|S )NrI   axis)rL   )r   r   )rO   r6   gluupdate_and_fetch_convrL   r(   padrP   rR   r:   rS   )r<   r>   caches      r-   rB   zConvolution.__call__V   s       #FF11 ++At||+DAq6DLL$,,#?HIA"OOAOOA  #r,   rA   )	r!   r"   r#   r   r5   r(   r)   rB   rC   rD   s   @r-   rF   rF   3   s+     
]  
D"(( 288 r,   rF   c            	            e Zd Zdef fdZ	 dded   deeeef      fdZ		 	 	 dde
j                  d	e
j                  dz  d
e
j                  dz  de
j                  fdZ xZS )ConformerBlockrG   c           
      j   t         |           |j                  |j                  z  }|| _        t        j                  |j                        | _        t        |j                  ||j                        | _
        t        j                  |j                        | _        |j                  dk(  rBt        |j                  |j                  |j                  |j                  |j                         n|j                  dk(  rmt#        |j                  |j                  |j                  |j                  |j                   |j$                  |j$                  d   |j$                  d   fnd      n!t'        |j                  |j                  d	      | _        t        j                  |j                        | _        t-        |      | _        t        j                  |j                        | _        t        |j                  ||j                        | _        t        j                  |j                        | _        y )
Nrel_posr3   r   r   rel_pos_local_attnr   r   rb   r3   r   r   context_sizeTr2   )r4   r5   r   r   rG   r6   	LayerNormnorm_feed_forward1r/   r   feed_forward1norm_self_attr   r
   r   r   r   r   r    r   	self_attn	norm_convrF   convnorm_feed_forward2feed_forward2norm_out)r<   rG   ff_hidden_dimr=   s      r-   r5   zConformerBlock.__init__k   s   t'?'??	"$,,t||"<(}dmmT\\$,,7 ((I5 *]]????$ ((,@@ 4]]????((4 #33A68M8Ma8PQ	 $) 	6 dll3%	"$,,t||"<(}dmmTT\\2r,   namer^   r`   normalrd   c                    |dk(  rtt        | j                  j                  | j                  j                  | j                  j                  | j                  j
                  | j                  j                        n|dk(  ryt        | j                  j                  | j                  j                  | j                  j                  | j                  j
                  | j                  j                  ||nd      n5t        | j                  j                  | j                  j                  d      }|j                  t        | j                  j                                      || _        y )Nr^   r_   r`   ra   rc   Tr2   )r
   rG   r   r   r   r   r   r   r   load_weightsr   ri   
parameters)r<   rp   rd   new_attns       r-   set_attention_modelz"ConformerBlock.set_attention_model   s    y  *		!!		!!YY''99//99//  ++ 4		!!		!!YY''99//99//-9-E\8 $		!!		!!% 	2 	l4>>+D+D+FGH!r,   Nr>   pos_embmaskr?   c           	      \   |d| j                  | j                  |            z  z   }| j                  |      }|| j                  ||||||      z   }|| j	                  | j                  |      |      z   }|d| j                  | j                  |            z  z   }| j                  |      S )Ng      ?)ry   rx   rZ   )rZ   )	rg   rf   rh   ri   rk   rj   rm   rl   rn   )r<   r>   rx   ry   rZ   x_norms         r-   rB   zConformerBlock.__call__   s     d(()@)@)CDDD##A&FFwe  
 
 		$..+5	99d(()@)@)CDDD}}Qr,   )   r}   )NNNr!   r"   r#   r   r5   r   r   tupler$   rw   r(   r)   rB   rC   rD   s   @r-   r\   r\   j   s    +3] +3` 3= "?@ " uS#X/ "J $( $ 88  D  hho	  
 r,   r\   c                       e Zd Zdef fdZdej                  dej                  fdZdej                  deej                  e	f   fdZ
dej                  dej                  deej                  ej                  f   fdZ xZS )	DwStridingSubsamplingrG   c                 F   t         |           |j                  dkD  r|j                  |j                  dz
  z  dk(  sJ |j                  | _        |j                  | _        t        t        j                  |j                  d            | _	        d| _
        d| _        | j                  dz
  dz  | _        d}|j                  }t        | j                        D ]T  }t        j                  |d| j                  z  z   | j                  z
  | j                  z        dz   }|dk  sKt!        d       t#        j$                  || j
                  | j                  | j                  | j                        t#        j&                         g| _        | j
                  }t        | j                  dz
        D ]  }| j(                  j+                  t#        j$                  ||| j                  | j                  | j                  |             | j(                  j+                  t#        j$                  || j
                  dddd             | j(                  j+                  t#        j&                                 t#        j,                  | j
                  |z  |j.                        | _        y )Nr   r   rI      z'Non-positive final frequency dimension!)in_channelsout_channelsrJ   rK   rL   )r   r   rJ   rK   rL   rM   )r4   r5   r   r   r   _conv_channelsr$   mathlog_sampling_num_stride_kernel_size_paddingr   rangefloor
ValueErrorr6   Conv2dReLUrk   appendr7   r   out)r<   rG   r   final_freq_dim_r=   s        r-   r5   zDwStridingSubsampling.__init__   sD    ##a'((D,C,Ca,GHQN	
O 150U0U-"<< $*A*A1!EF**Q.14t))*A

#a$--&77$:K:KKll# 	  ! !JKK + II'!00 --|| GGI	
	 ))t))A-.AII		 +!, $ 1 1<< MM&	 II		 +!%!4!4 !	 IIRWWY'+ /. 99T00>A4<<Pr,   r>   r?   c                 x    |j                  d      }| j                  D ]
  } ||      } |j                  d      S )N)r   rI   r   r   )r   r   r   rI   )	transposerk   )r<   r>   layers      r-   conv_forwardz"DwStridingSubsampling.conv_forward  s6    KK%YYEaA {{<((r,   c                    |j                   d   }|dk(  r|dfS | j                  dkD  r| j                  }ndd| j                  z  | j                  z  | j                  z  }t	        j
                  t	        j                  |j                  |z  d            }d|z  }||z  }|dk(  r|dfS t        j                  t        j                  ||d      D cg c]  }| j                  |       c}      dfS c c}w )Nr   r   F        rI   T)shaper   r   r   r   ceilr   sizer(   concatsplitr   )r<   r>   bcfx_ceilpnew_batch_sizechunks           r-   conv_split_by_batchz)DwStridingSubsampling.conv_split_by_batch  s    GGAJ6e8O001466BT0004<<?$,,NF		$((166F?A67AdBbQe8Oyy3588A~q3QR3Q%Tu%3QR
 	Rs   C0lengthsc                    t        | j                        D ]D  }t        j                  |d| j                  z  z   | j
                  z
  | j                  z        dz   }F |j                  t        j                        }t        j                  |d      }| j                  dk7  r| j                  dk(  r9d| j                  z  | j                  z  | j                  z  }|j                  |kD  }nd}|r(| j                  |      \  }}|s5| j                  |      }n#| j                  |      }n| j                  |      }|j                  dd      j!                  |j"                  d   |j"                  d   d      }| j%                  |      }||fS )	NrI   g      ?r   rU   rb   r   Tr   )r   r   r(   r   r   r   r   astypeint32expand_dimsr   r   r   r   r   swapaxesreshaper   r   )r<   r>   r   r   r   need_to_splitsuccesss          r-   rB   zDwStridingSubsampling.__call__)  s[   t))*Aq4==0043D3DDT   + ..*NN11%00B6449!4!44t||CdllR ! $!55a8
7))!,A%%a(!!!$AJJq!$$QWWQZR@HHQK'zr,   )r!   r"   r#   r   r5   r(   r)   r   r   r'   r   rB   rC   rD   s   @r-   r   r      s    >Q] >Q@)bhh )288 )RXX %$2G ("(( RXX %"((@R:S r,   r   c            	            e Zd Zdef fdZ	 dded   deeeef      fdZ		 dde
j                  d	e
j                  dz  d
ee
j                  e
j                  f   fdZ xZS )	ConformerrG   c                 $   t         |           || _        |j                  dk(  r2t	        |j
                  |j                  |j                        | _        ns|j                  dk(  r]t        |j
                  |j                  |j                  |j                  |j                  d   |j                  d   fnd      | _        nd | _        |j                  dkD  rR|j                  dk(  r|j                  d	u rt        |      | _        nSt!        j"                         | _        t%        d
      t!        j&                  |j(                  |j
                        | _        t+        |j,                        D cg c]  }t/        |       c}| _        y c c}w )Nr^   r   max_lenscale_inputr`   r   r   ra   r   r   r   rd   dw_stridingFz/Other subsampling haven't been implemented yet!)r4   r5   rG   r   r	   r   r   r   pos_encr   r    r   r   r   r   
pre_encoder6   IdentityNotImplementedErrorr7   r   r   r   r\   layers)r<   rG   r   r=   s      r-   r5   zConformer.__init__L  sJ   	$$	10,, MMDL
 &&*>>5,, MM((4 #33A68M8Ma8PQDL  DL""Q&=0T5M5MQV5V"7"="$++-)E  !iidllCDO5:4==5IJ5I~d+5IJJs   2Frp   rq   rd   c                    |dk(  rPt        | j                  j                  | j                  j                  | j                  j                        | _        na|dk(  rUt        | j                  j                  | j                  j                  | j                  j                  |r|nd      | _        nd | _        | j                  D ]  }|j                  ||        y )Nr^   r   r`   ra   r   )	r	   rG   r   r   r   r   r   r   rw   )r<   rp   rd   r   s       r-   rw   zConformer.set_attention_modelp  s    
 90		))		11 II..DL
 ))5		))		11 II..-9\x	DL  DL[[E%%dL9 !r,   Nr>   r   r?   c                 n   |At        j                  |j                  d   f|j                  d   t         j                        }t	        | j
                  t              r| j                  ||      \  }}nCt	        | j
                  t        j                        r| j                  |      }|}nt        d      |d gt        | j                        z  }d }| j                  *| j                  ||d   |d   j                  nd      \  }}t        | j                  |      D ]  \  }} ||||      } ||fS )Nr   )dtypez(Non-implemented pre-encoding layer type!)offset)rx   rZ   )r(   fullr   int64
isinstancer   r   r6   r7   r   lenr   r   r   zip)r<   r>   r   rZ   out_lengthsrx   r   cs           r-   rB   zConformer.__call__  s    ?gghhG doo'<=!__Q8NA{3"A!K%&PQQ=FS--E<<#*/(*>uQxA & JAw
 DKK/HE1a2A 0 +~r,   r|   )NNr~   rD   s   @r-   r   r   K  s    "K] "KN 3=:?@: uS#X/:2 CG$&HHtO	rxx!	"r,   r   )r   dataclassesr   typingr   r   mlx.corecorer(   mlx.nnr6   mlx.nn.utilsr   parakeet_mlx.attentionr   r   r	   r
   r   r   Moduler/   rF   r\   r   r   r+   r,   r-   <module>r      s     ! $   %  1 1 1*>")) >4")) 4na RYY a HzBII zz\		 \r,   