
    hJJj`)                        d dl Z d dlZd dlmZ d dlmZmZmZ d dlZ	d dl
Z
d dlmc mZ d dl
mZmZ e G d d             Z G d dej"                        Z G d	 d
ej$                        Z G d dej&                        ZddZ G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Zy)    N)	dataclass)DictIterableOptional)Tensornnc                   r    e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   y)ModelDimensionsn_melsn_audio_ctxn_audio_staten_audio_headn_audio_layern_vocab
n_text_ctxn_text_staten_text_headn_text_layerN)__name__
__module____qualname__int__annotations__     c/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_whisper/torch_whisper.pyr
   r
      s;    KLOr   r
   c                   (     e Zd Zdedef fdZ xZS )	LayerNormxreturnc                 p    t         |   |j                               j                  |j                        S N)superforwardfloattypedtype)selfr   	__class__s     r   r$   zLayerNorm.forward   s'    wqwwy)..qww77r   )r   r   r   r   r$   __classcell__r)   s   @r   r   r      s    8 8F 8 8r   r   c                       e Zd ZdedefdZy)Linearr   r    c                     t        j                  || j                  j                  |j                        | j
                  d       S | j
                  j                  |j                              S r"   )Flinearweighttor'   bias)r(   r   s     r   r$   zLinear.forward"   sV    xxKKNN177#II%D
 	
 ,099<<+@
 	
r   N)r   r   r   r   r$   r   r   r   r-   r-   !   s    
 
F 
r   r-   c                   6     e Zd Zdededee   def fdZ xZS )Conv1dr   r1   r3   r    c                     t         |   ||j                  |j                        |d       S |j                  |j                              S r"   )r#   _conv_forwardr2   r'   )r(   r   r1   r3   r)   s       r   r7   zConv1d._conv_forward+   sI     w$vyy!4<4
 	
=AWWQWW=M
 	
r   )r   r   r   r   r   r7   r*   r+   s   @r   r5   r5   *   s2    

!'
/7/?
	
 
r   r5   c                    |dz  dk(  sJ t        j                  |      |dz  dz
  z  }t        j                  | t        j                  |dz        z        }t        j                  |       ddt         j
                  f   |t         j
                  ddf   z  }t        j                  t        j                  |      t        j                  |      gd      S )z*Returns sinusoids for positional embedding   r      Ndim)	nplogtorchexparangenewaxiscatsincos)lengthchannelsmax_timescalelog_timescale_incrementinv_timescalesscaled_times         r   	sinusoidsrL   3   s    a<1 ff]3x1}q7HIYY 77%,,xST}:UUVN,,v&q"**}5rzzST}8UUK99eii,eii.DE1MMr   c            
       |     e Zd Zdedef fdZ	 	 	 ddedee   dee   dee   fdZ	 dd	ed
ededee   fdZ	 xZ
S )MultiHeadAttentionn_staten_headc                     t         |           || _        t        ||      | _        t        ||d      | _        t        ||      | _        t        ||      | _        y )NF)r3   )r#   __init__rP   r-   querykeyvalueout)r(   rO   rP   r)   s      r   rR   zMultiHeadAttention.__init__=   sO    GW-
'77GW-
'7+r   r   xamaskkv_cachec                 .   | j                  |      }||| j                  |vr+| j                  ||n|      }| j                  ||n|      }n|| j                     }|| j                     }| j                  ||||      \  }}	| j	                  |      |	fS r"   )rS   rT   rU   qkv_attentionrV   )
r(   r   rW   rX   rY   qkvwvqks
             r   r$   zMultiHeadAttention.forwardE   s     JJqMrzTXXX-E bjb1A


13A "A$A##Aq!T2Bxx|Rr   r\   r]   r^   c                    |j                   \  }}}|| j                  z  dz  } |j                  g |j                   d d | j                  d j                  dddd      |z  } |j                  g |j                   d d | j                  d j                  dddd      |z  } |j                  g |j                   d d | j                  d j                  dddd      }||z  }	||	|d |d |f   z   }	|	j	                         }	t        j                  |	d      j                  |j                        }
|
|z  j                  dddd      j                  d      |	j                         fS )	Ng      пr9   r   r:      r;   )	start_dim)shaperP   viewpermuter%   r/   softmaxr2   r'   flattendetach)r(   r\   r]   r^   rX   n_batchn_ctxrO   scaler`   ws              r   r[   z MultiHeadAttention.qkv_attention[   sk    #$''DKK'E1AFF1AGGBQK11b199!Q1EMAFF1AGGBQK11b199!Q1EMAFF1AGGBQK11b199!Q1EUd6E66E6>**BXXZIIbb!$$QWW-Aq!Q*22Q2?LLr   NNNr"   )r   r   r   r   rR   r   r   dictr$   r[   r*   r+   s   @r   rN   rN   <   s    , ,S ,  $!%#'   V  v	 
 4. . IMMM"M'-M5=f5EMr   rN   c            
       `     e Zd Zd
dededef fdZ	 	 	 ddedee   dee   dee   fd	Z	 xZ
S )ResidualAttentionBlockrO   rP   cross_attentionc                 d   t         |           t        ||      | _        t	        |      | _        |rt        ||      nd | _        |rt	        |      nd | _        |dz  }t        j                  t        ||      t        j                         t        ||            | _        t	        |      | _        y )N   )r#   rR   rN   attnr   attn_ln
cross_attncross_attn_lnr   
Sequentialr-   GELUmlpmlp_ln)r(   rO   rP   rs   n_mlpr)   s        r   rR   zResidualAttentionBlock.__init__n   s    &w7	 ) 4Cw/ 	 4CYw/!==7E"BGGIveW/E
  (r   r   rW   rX   rY   c                    || j                  | j                  |      ||      d   z   }| j                  r)|| j                  | j                  |      ||      d   z   }|| j	                  | j                  |            z   }|S )NrX   rY   r   )rY   )rv   rw   rx   ry   r|   r}   )r(   r   rW   rX   rY   s        r   r$   zResidualAttentionBlock.forward   s{     		$,,q/x	HKK??DOOD$6$6q$92OQRSTTAQ((r   )Fro   )r   r   r   r   boolrR   r   r   rp   r$   r*   r+   s   @r   rr   rr   m   sa    ) )S )4 )(  $!%#' V v	
 4.r   rr   c            
       @     e Zd Zdededededef
 fdZdefdZ xZS )	AudioEncoderr   rl   rO   rP   n_layerc           	      L   t         |           t        ||dd      | _        t        ||ddd      | _        | j                  dt        ||             t        j                  t        |      D cg c]  }t        ||       c}      | _        t        |      | _        y c c}w )Nrc   r:   )kernel_sizepaddingr9   )r   strider   positional_embedding)r#   rR   r5   conv1conv2register_bufferrL   r   
ModuleListrangerr   blocksr   ln_post)r(   r   rl   rO   rP   r   _r)   s          r   rR   zAudioEncoder.__init__   s     	FGAF
GW!AqQ
3Yug5NO8:>CGnMn#GV4nM9
 !) Ns   1B!r   c                    t        j                  | j                  |            }t        j                  | j                  |            }|j	                  ddd      }|j
                  dd | j                  j
                  k(  sJ d       || j                  z   j                  |j                        }| j                  D ]
  } ||      } | j                  |      }|S )zt
        x : torch.Tensor, shape = (batch_size, n_mels, n_ctx)
            the mel spectrogram of the audio
        r   r9   r:   Nzincorrect audio shape)r/   gelur   r   rg   re   r   r2   r'   r   r   )r(   r   blocks      r   r$   zAudioEncoder.forward   s    
 FF4::a=!FF4::a=!IIaAwwqr{d77===V?VV=***..qww7[[EaA ! LLOr   )r   r   r   r   rR   r   r$   r*   r+   s   @r   r   r      s<    **"%*03*=@*KN* r   r   c            
       P     e Zd Zdededededef
 fdZddeded	ee   fd
Z xZ	S )TextDecoderr   rl   rO   rP   r   c           
         t         |           t        j                  ||      | _        t        j
                  t        j                  ||            | _        t        j                  t        |      D cg c]  }t        ||d       c}      | _        t        |      | _        t        j                  ||      j                  t         j"                         j%                  d      }| j'                  d|d       y c c}w )NT)rs   r:   rX   F
persistent)r#   rR   r   	Embeddingtoken_embedding	Parameterr?   emptyr   r   r   rr   r   r   lnfill_r=   inftriu_r   )	r(   r   rl   rO   rP   r   r   rX   r)   s	           r   rR   zTextDecoder.__init__   s     	!||GW=$&LLUG1L$M!8: w'A 'wM'9
 G${{5%(..w7==a@VTe<s   5C>r   rW   rY   c                    |r/t        t        |j                                     j                  d   nd}| j	                  |      | j
                  |||j                  d   z    z   }|j                  |j                        }| j                  D ]  } |||| j                  |      } | j                  |      }|t        j                  | j                  j                  j                  |j                        dd      z  j                         }|S )z
        x : torch.LongTensor, shape = (batch_size, <= n_ctx)
            the text tokens
        xa : torch.Tensor, shape = (batch_size, n_audio_ctx, n_audio_state)
            the encoded audio features to be attended on
        r:   r   rb   r   )nextitervaluesre   r   r   r2   r'   r   rX   r   r?   	transposer1   r%   )r(   r   rW   rY   offsetr   logitss          r   r$   zTextDecoder.forward   s     <Dd8??,-.44Q7  #''!''"+1EFG 	
 DDN[[Ea$))h?A ! GGAJ 4 4 ; ; > >qww GANN
%' 	 r   r"   )
r   r   r   r   rR   r   r   rp   r$   r*   r+   s   @r   r   r      sN    ==#&=14=>A=LO=& V x~ r   r   c                   &    e Zd Zdef fdZdefdZdej                  fdZ	dej                  dej                  fd	Z
dej                  dej                  d
eeej                  f   fdZed        Zed        Zed        Zddee   fdZ xZS )Whisperdimsc                 (   t         |           || _        t        | j                  j                  | j                  j
                  | j                  j                  | j                  j                  | j                  j                        | _	        t        | j                  j                  | j                  j                  | j                  j                  | j                  j                  | j                  j                        | _        t#        j$                  | j                  j                  | j                  j                  t"        j&                        }d|| j                  j                  dz  d  | j)                  d|j+                         d       y )Nr'   Tr9   alignment_headsFr   )r#   rR   r   r   r   r   r   r   r   encoderr   r   r   r   r   r   decoderr?   zerosr   r   	to_sparse)r(   r   	all_headsr)   s      r   rR   zWhisper.__init__   s   	#IIII!!II##II""II##
 #IIII  II""II!!II""
 KKII""DII$9$9
	 48	$))((A-/0.	0C0C0ERWXr   dumpc                    t        j                  t        j                  t	        j
                  |            t              j                         }t        j                  |      j                  | j                  j                  | j                  j                        }| j                  d|j                         d       y )Nr   r   Fr   )r=   
frombuffergzip
decompressbase64	b85decoder   copyr?   
from_numpyreshaper   r   r   r   r   )r(   r   arrayrX   s       r   set_alignment_headszWhisper.set_alignment_heads   s    OOF,,T234

$& 	 &..II""DII$9$9
 	.0@USr   melc                 $    | j                  |      S r"   )r   )r(   r   s     r   embed_audiozWhisper.embed_audio   s    ||C  r   tokensaudio_featuresc                 &    | j                  ||      S r"   )r   )r(   r   r   s      r   r   zWhisper.logits  s    ||FN33r   r    c                 D    | j                  || j                  |            S r"   )r   r   )r(   r   r   s      r   r$   zWhisper.forward  s     ||FDLL$566r   c                 H    t        | j                               j                  S r"   )r   
parametersdevicer(   s    r   r   zWhisper.device	  s    DOO%&---r   c                 4    | j                   j                  dk\  S )Ni  )r   r   r   s    r   is_multilingualzWhisper.is_multilingual  s    yy  E))r   c                 `    | j                   j                  dz
  t        | j                        z
  S )Ni5  )r   r   r   r   r   s    r   num_languageszWhisper.num_languages  s'    yy  5(3t/C/C+DDDr   cachec                      i ni g  fddt         j                  ffd} j                  j                  |       fS )a  
        The `MultiHeadAttention` module optionally accepts `kv_cache` which stores the key and value
        tensors calculated for the previous positions. This method returns a dictionary that stores
        all caches, and the necessary hooks for the key and value projection modules that save the
        intermediate tensors to be reused during later calculations.

        Returns
        -------
        cache : Dict[nn.Module, torch.Tensor]
            A dictionary object mapping the key/value projection modules to its cache
        hooks : List[RemovableHandle]
            List of PyTorch RemovableHandle objects to stop the hooks to be called
        c                     | vs&|j                   d   j                  j                  kD  r
|| <   |    S t        j                  |    |gd      j                         | <   |    S )Nr:   r;   )re   r   r   r?   rC   rj   )moduler   outputr   r(   s      r   save_to_cachez5Whisper.install_kv_cache_hooks.<locals>.save_to_cache&  sk    U"fll1o		8L8L&L &f =  !&		5=&*Aq I P P Rf= r   layerc                     t        | t              rUj                  | j                  j	                               j                  | j
                  j	                               y y r"   )
isinstancerN   appendrT   register_forward_hookrU   )r   hooksr   s    r   install_hooksz5Whisper.install_kv_cache_hooks.<locals>.install_hooks.  sI    %!34UYY<<]KLU[[>>}MN 5r   )r   Moduler   apply)r(   r   r   r   r   s   `` @@r   install_kv_cache_hookszWhisper.install_kv_cache_hooks  sO     #.	5	B	!	O 	O
 	=)e|r   r"   )r   r   r   r
   rR   bytesr   r?   r   r   r   r   strr$   propertyr   r   r   r   rp   r   r*   r+   s   @r   r   r      s    Y_ Y2T T!u|| !4U\\ 45<< 47<<7).7	c5<<	 7
 . . * * E EHTN r   r   )i'  )r   r   dataclassesr   typingr   r   r   numpyr=   r?   torch.nn.functionalr   
functionalr/   r   r
   r   r-   r5   rL   r   rN   rr   r   r   r   r   r   r   <module>r      s      ! + +      
 
 
8 8

RYY 

RYY 
N.M .MbRYY @299 B*")) *ZYbii Yr   