
    (HJj(                        d dl Z d dlmZmZmZ d dlmZ d dlm	Z	  G d de	j                        Z G d de	j                        Z G d de	j                        Z G d	 d
e	j                        Z	 	 ddee   dee   fdZy)    N)ListOptionalUnionc                        e Zd Z	 	 	 	 	 	 	 ddededededeee   ef   deee   ef   dedef fd	Zdd
eeej                  f   fdZ
 xZS )SuScaledRoPEdimsbasemax_position_embeddings original_max_position_embeddingsshort_factorlong_factorshort_mscalelong_mscalec	                 D   t         |           | _        || _        |t	        j
                  d|dt        j                        |z  z  }	t	        j                  |t        j                        |	z  | _        fd}
|z  }|xs |dk  rd| _	        y |
|      | _	        y)a  
        Su Scaled Rotary Embedding layer.

        Args:
            dims (int): The feature dimensions to be rotated.
            base (int, optional): Base for the exponential scaling.
            max_position_embeddings (int, optional): The maximum sequence
              length that this model was trained with. This is used to determine
              the size of the original RoPE embeddings when using long scaling.
              Default: ``131072``.
            original_max_position_embeddings (int, optional): The maximum
              sequence length that this model was trained with. This is used to
              determine the size of the original RoPE embeddings when using long
              scaling. Default: ``4096``.
            short_factor (float or list[float], optional): List of scaling
              factors for sequences of length lesser than
              ``original_max_position_embeddings``. Default: ``1.0``.
            long_factor (float or list[float], optional): List of scaling
              factors for sequences of length greater than
              ``original_max_position_embeddings``.  Default: ``1.0``.
            short_mscale (float, optional): Scale the input prior to embedding.
            long_mscale (float, optional): Scale the input prior to embedding.
        r      dtypec                     t        j                  dt        j                  |       t        j                        z  z         S )N   )mathsqrtlog)factorr   s    b/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/rope_utils.pydefault_scalez,SuScaledRoPE.__init__.<locals>.default_scale4   s4    99DHHV$txx0P'QQQ           ?N)
super__init__r   dimmxarangefloat32array_freqs_scale)selfr   r	   r
   r   r   r   r   r   freqsr   r   	__class__s       `       r   r   zSuScaledRoPE.__init__   s    D 	0P-1dARZZ@4GHhh{"**=E	
 )+KK!VVs]cf@Ur   offsetc           	          |d   }| j                   |dd | j                  f   z  |dd | j                  f<   t        j                  j	                  || j                  dd d|| j
                        S )N.Fr   traditionalr	   scaler*   r(   )r&   r    r!   fastroper%   r'   xr*   s      r   __call__zSuScaledRoPE.__call__<   sr    cF![[1S*DHH*_+==#zz/ww||HH++  
 	
r   )     @i      r   r   NNr   )__name__
__module____qualname__intfloatr   r   r   r!   r$   r3   __classcell__r)   s   @r   r   r   
   s     '-042514"!/W/W /W "%	/W
 +./W DK.//W 4;-./W /W /Wb
%RXX"6 
r   r   c                   R     e Zd Z	 	 	 	 d
dededededef
 fdZd Zddefd	Z	 xZ
S )
Llama3RoPEr   r
   r-   r	   scaling_configc                    t         |           || _        || _        || _        |d   }|j                  dd      }|j                  dd      }|j                  dd      }	|	|z  }
|	|z  }|t        j                  d|d	      |z  z  }d	t        j                  z  |z  }t        j                  ||
kD  ||z  |      }||kD  ||
k  z  }|	|z  |z
  ||z
  z  }|d
|z
  |z  |z   z  }t        j                  |||      | _
        y )Nr   low_freq_factorr   high_freq_factorg      @r   i    r   r   r   )r   r   r   r
   r-   getr!   r"   piwherer%   )r'   r   r
   r-   r	   r@   r   rB   rC   old_context_lenlow_freq_wavelenhigh_freq_wavelenr(   wavelensis_medium_freqsmooth_factorssmooth_freqsr)   s                    r   r   zLlama3RoPE.__init__K   s2    		'>$&)(,,->D)--.@#F(,,.

 +_<+.>>1dA.56ruu9u$$44efneL"%668FV;VW)H4F.
 ^!3v = NOhh~|UCr   c                 R    | j                    d| j                   d| j                   S )Nz, traditional=z, max_position_embeddings=)r   r-   r
   )r'   s    r   
extra_reprzLlama3RoPE.extra_reprn   s4    yyk(8(8'9 :''+'C'C&DF	
r   r*   c           	          t         j                  j                  || j                  | j                  d d|| j
                        S Nr   r,   r!   r/   r0   r   r-   r%   r1   s      r   r3   zLlama3RoPE.__call__t   ?    ww||II((++  
 	
r   )   F'  Nr6   )r7   r8   r9   r:   boolr;   dictr   rO   r3   r<   r=   s   @r   r?   r?   J   s^     (,!#!D!D "%!D 	!D
 !D !DF
	
# 	
r   r?   c                   :     e Zd Z	 	 	 	 	 	 	 	 	 d fd	ZddZ xZS )YarnRoPEc                 p   t         |           fdfd}d
d}d } |||	       |||
      z  | _        t        j                  ddt        j
                        z  z  }||z  } |       \  }}d	 |||dz        z
  }||z  ||z  |d|z
  z  z   z  | _        | _        || _        y )Nc                     t        j                  | dz  t         j                  z  z        z  dt        j                        z  z  S )Nr   )r   r   rE   )num_rotationsr	   r   r   s    r   yarn_find_correction_dimz3YarnRoPE.__init__.<locals>.yarn_find_correction_dim   sI    ((48IDGG8ST TXXd^#% %r   c                      t        j                               } t        j                               }t        | d      t	        |dz
        fS )Nr   r   )r   floorceilmaxmin)lowhigh	beta_fast	beta_slowr   r]   s     r   yarn_find_correction_rangez5YarnRoPE.__init__.<locals>.yarn_find_correction_range   sG    **5i@AC995i@ADsA;D$( 333r   r   c                 J    | dk  ryd|z  t        j                  |       z  dz   S )Nr   r   g?)r   r   )r.   mscales     r   yarn_get_mscalez*YarnRoPE.__init__.<locals>.yarn_get_mscale   s(    z<$((5/1C77r   c                     | |k(  r|dz  }t        j                  |t         j                        | z
  || z
  z  }t        j                  |dd      S )NgMbP?r   r   r   )r!   r"   r#   clip)min_valmax_valr    linear_funcs       r   yarn_linear_ramp_maskz0YarnRoPE.__init__.<locals>.yarn_linear_ramp_mask   sO    '!5 99S

;gE'!K 77;1--r   r   r   r   r   )r   r   )	r   r   ri   r!   r"   r#   r%   r   r-   )r'   r   r-   r
   r	   scaling_factorr   re   rf   ri   mscale_all_dimrg   rj   rp   
freq_extra
freq_interrc   rd   	freq_maskr]   r)   s    `  ` ```          @r   r   zYarnRoPE.__init__   s     		%	4
	8
	. &nf=NA
 
 bii4"**ELM
#j0
.0	T/T419EE	!J."Z1y=%AA
 	&r   c           	         | j                   dk7  r6|d   }| j                   |dd | j                  f   z  |dd | j                  f<   t        j                  j	                  || j                  | j
                  d d|| j                        S )Nr   .r,   )ri   r   r!   r/   r0   r-   r%   r1   s      r   r3   zYarnRoPE.__call__   s    ;;##A"&++#{{2B0C"CAc;TYY;ww||II((++  
 	
r   )	FrT   rU   r   r5       r   r   r   r6   )r7   r8   r9   r   r3   r<   r=   s   @r   rY   rY      s+      $)-5'n
r   rY   c                   D     e Zd Z	 	 	 ddededededef
 fdZd	dZ xZS )
ProportionalRoPEr   rotated_dimsr-   r	   r   c                 H   t         |           || _        || _        ||kD  rt	        d      t        j                  d|dt
        j                        |z  }t        j                  |||z  z  t        j                  ||z
  dz  ft
        j                        g      | _        y )Nz(rotated_dims should be smaller than dimsr   r   r   )r   r   r   r-   
ValueErrorr!   r"   r#   concatenatefullinfr%   )r'   r   rz   r-   r	   r   	exponentsr)   s          r   r   zProportionalRoPE.__init__   s     		&$GHHIIaq

CdJ	nn$	/*$-!35rvv>
r   c           	          t         j                  j                  || j                  | j                  d d|| j
                        S rQ   rR   r1   s      r   r3   zProportionalRoPE.__call__   rS   r   )Fr4   r   r6   )	r7   r8   r9   r:   rV   r;   r   r3   r<   r=   s   @r   ry   ry      sH    
 "

 
 	

 
 
.	
r   ry   r@   r
   c           
         |&|j                  d      xs |j                  dd      }nd}|dv r(|dk(  rd|d   z  nd}t        j                  | |||	      S |d
k(  rt        | ||||      S |dv r-|d   }dD ci c]  }||v r|||    }	}t	        d| ||||d|	S |dk(  rt        | |||d   |d   |d         S |dk(  r<t        | t        | |j                  dd      z        |||j                  dd            S |dk(  rL|j                  dg       }
t        |
      dk(  sJ dt        |
       d       t        j                  | ||      S t        d|       c c}w )Ntype	rope_typedefault)r   linearr   r   r   r   )r-   r	   r.   llama3)r   r
   r-   r	   r@   )yarndeepseek_yarnztelechat3-yarn)r   re   rf   ri   rr   )r   r
   r-   rq   r	   longroper   r   r   )r   r	   r
   r   r   r   proportionalpartial_rotary_factor)r   rz   r-   r	   r   mropemrope_section   z.MRoPE currently only supports 3 sections, got .)r-   r	   zUnsupported RoPE type  )
rD   nnRoPEr?   rY   r   ry   r:   lenr|   )r   r	   r-   r@   r
   r   r.   rq   keyrope_kwargsr   s              r   initialize_roper      s    !"&&v. 
.2D2D3
	 	))09X0EN8,,3wwt4uMM	h	$;#)
 	
 
A	A'1

 n$ $$ 	 

  
$;#)
 
 	
 
j	 $;-;2. (7&}5	
 		
 
n	$TN$6$67NPS$TTU#!%%h4
 	
 
g	&**?B?!#	R;C<N;OqQ	R#wwt4@@1)=>>[

s   9E)NN)r   typingr   r   r   mlx.corecorer!   mlx.nnr   Moduler   r?   rY   ry   rW   r:   r   r   r   r   <module>r      s     ( (  =
299 =
@3
 3
lD
ryy D
N!
ryy !
P &*-1I? TN	I?
 &c]I?r   