
    $HJj.z                        d dl mZ d dlmZ ddlmZmZ ddlmZ ddlm	Z	m
Z
mZmZ ddlZddlZddlZddlZddlZddlZd:de	e   d	e
e   d
efdZ ed      Zd Ze G d d             Zd Z e       Z[ ej:                  d      Z ed      Z ed      Z  ed e D              Z! eg d      Z"dd e#d       e#d       e#d      dZ$ddg e%dd       e%dd       Z& ed!      Z' ed"      Z(d#d$d%d&Z) eg d'      Z*d(d)d*Z+d+d,d-d.d/Z, ed0      Z- ed1      Z.d2Z/e/d    Z0e/d   Z1 ed3      Z2d4 Z3d5 Z4 G d6 d7      Z5 G d8 d9      Z6y);   )data)MToken    )	dataclassreplace)	num2words)ListOptionalTupleUnionNtokensunkreturnc                    | D ch c]/  }|j                   j                  |j                   j                  1 }}| D ch c]/  }|j                   j                  |j                   j                  1 }}| D ch c]  }|j                   j                   }}|d }nbd}| D ][  }|j                   j                  r&|r$|d   j                         s|j                  r|dz  }||j                  |n|j                  z  }] t        dj                  d | d d D              | d   j                  z   t        | d       j                  | d   j                  || d   j                  | d   j                  t        j                  | d   j                   j                   d t#        |      dk(  rt%        |      d   nd |rt        |      nd dj                  t'        | D ch c]  }|j                   j(                  D ]  }| ! c}}            | d   j                   j                  d |v rd n
t+        |      	      
      S c c}w c c}w c c}w c c}}w )N  c              3   N   K   | ]  }|j                   |j                  z     y wNtext
whitespace.0tks     S/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/misaki/en.py	<genexpr>zmerge_tokens.<locals>.<genexpr>   s     C{RWWr}},{   #%c                 :    t        d | j                  D              S )Nc              3   J   K   | ]  }||j                         k(  rd nd  yw)r      N)lowerr   cs     r   r   z1merge_tokens.<locals>.<lambda>.<locals>.<genexpr>   s"     *Ww!QWWY1A+Ews   !#)sumr   )r   s    r   <lambda>zmerge_tokens.<locals>.<lambda>   s    s*Wrww*W'W    keyr   r   )is_headaliasstresscurrency	num_flagsprespacerating)r   tagr   phonemesstart_tsend_ts_)r5   r,   r-   r0   r/   isspacer2   r   joinr   maxr1   r   r3   r4   
Underscorer*   lenlistsortedr.   min)r   r   r   r,   r-   r0   r2   r$   s           r   merge_tokensr>      s   $*FFbbddkk.EbddkkFFF(.L""$$--2KHL$*+FbbddkkFF+
{Btt}}(2,2F2F2HR[[Cr{{2CH  WWCvcr{CCfRjooUWX\\":((##bz  


1IKK''&)&kQ&64<?D&.S]Dggf&%Q&B"$$..Qa.a&%QRSAY[[))6>4s6{
  GL+* &Rs!   H:H:H?H?9I$I	*u
   AIOQWYʤʧc                 .    | rt        d | D              S dS )Nc              3   4   K   | ]  }|t         v rd nd  yw)r!   r   N)
DIPHTHONGSr#   s     r   r   z stress_weight.<locals>.<genexpr>-   s     7BqAOq*B   r   )r%   )pss    r   stress_weightrD   ,   s    ;=37B77D1Dr'   c                   0    e Zd ZU dZee   ed<   dZeed<   y)TokenContextNfuture_vowelF	future_to)__name__
__module____qualname__rG   r
   bool__annotations__rH    r'   r   rF   rF   /   s    #'L(4.'Itr'   rF   c                  <     dd l j                  d        fdS )Nr   u   ^['‘’]+|\p{Lu}(?=\p{Lu}\p{Ll})|(?:^-)?(?:\d?[,.]?\d)+|[-_]+|['‘’]{2,}|\p{L}*?(?:['‘’]\p{L})*?\p{Ll}(?=\p{Lu})|\p{L}+(?:['‘’]\p{L})*|[^-_\p{L}'‘’\d]|['‘’]+$c                 (    j                  |       S r   )findall)wordSUBTOKEN_REGEXregexs    r   r&   z'make_subtokenize_once.<locals>.<lambda>8   s    ~t<r'   )rT   compile)rS   rT   s   @@r   make_subtokenize_oncerV   5   s!    ]]  $Z  [N<=r'   z\[([^\]]+)\]\(([^\)]*)\)u   ',-._‘’/u   ;:,.!?—…"“”c              #   *   K   | ]  }|d vs|  yw)u   "“”NrN   )r   ps     r   r   r   A   s     E1!92DQs   	).,-LRB--RRB-``""'':$#NFP()i   i   )r[   r\   r]   r^   r_   '   -   A   [   a   {   u#   bdfhjklmnpstvwzðŋɡɹɾʃʒʤʧθu   AIOWYiuæɑəɛɪɹʊʌ)dollarcent)poundpence)eurorm   )ra      £u   €)stndrdthdotslash)rY   /percentandplusat)%&+@uH   AIOWYbdfhijklmnpstuvwzæðŋɑɔəɛɜɡɪɹɾʃʊʌʒʤʧˈˌθᵊᵻʔuD   AIQWYabdfhijklmnpstuvwzðŋɑɒɔəɛɜɡɪɹʃʊʌʒʤʧˈˌːθᵊu   ˌˈu    AIOQWYaiuæɑɒɔəɛɜɪʊʌᵻc                 f    d }| S |dk  r* j                  t        d      j                  t        d      S |dk(  s|dv r6t         v r. j                  t        d      j                  t        t              S |dv rAt         fdt        D              r)t         fdt
        D              r S  |t         z         S |dk\  r*t         vr"t         v r j                  t        t              S |dkD  rAt         fd	t        D              r)t         fd
t
        D              r S  |t         z         S  S )Nc                 V   t        t        |             }|D ci c]$  \  }}|t        v s|t        d ||d  D              & }}}|j	                         D ]  \  }}||   \  }}|dz
  |f||<    dj                  t        |      D cg c]  \  }}|	 c}}      } | S c c}}w c c}}w )Nc              3   8   K   | ]  \  }}|t         v s|  y wr   )VOWELS)r   jvs      r   r   z1apply_stress.<locals>.restress.<locals>.<genexpr>`   s     A'$!QQ&[A's         ?r   )r;   	enumerateSTRESSESnextitemsr7   r<   )rC   ipsirX   stressesr   r5   ss           r   restresszapply_stress.<locals>.restress^   s    9R=!NQccdaUVZbUbAtA#ab'AAAccNN$DAqq6DAq#gq\CF % WWF3K0KDAqaK01	 d 1s   BB	B%
r   r   )r         )r   r   r   c              3   &   K   | ]  }|v 
 y wr   rN   r   r   rC   s     r   r   zapply_stress.<locals>.<genexpr>l   s     &EHqq{H   c              3   &   K   | ]  }|v 
 y wr   rN   r   r   rC   s     r   r   zapply_stress.<locals>.<genexpr>m        +Fqq{Fr   r   c              3   &   K   | ]  }|v 
 y wr   rN   r   s     r   r   zapply_stress.<locals>.<genexpr>r   s     :AARKr   c              3   &   K   | ]  }|v 
 y wr   rN   r   s     r   r   zapply_stress.<locals>.<genexpr>s   r   r   )r   PRIMARY_STRESSSECONDARY_STRESSallr   r   )rC   r,   r   s   `  r   apply_stressr   ]   s    ~		"zz."-556FKK	2&I-.B2Fzz*B/77HXYY	;	3&EH&E#E+F++I(2-..	1r16F"6Lzz*N;;	!:::+F++I+,,Ir'   c                 @    t        t        j                  d|             S )Nz^[0-9]+$)rL   rematch)r   s    r   is_digitr   x   s    d+,,r'   c                       e Zd Zed        Zd Zd Zd Zed        Zd Z	d Z
d Zd	 Zd
 Zd Zd Zd Zd Zed        Zd Zd Zed        Zed        Zd Zy)Lexiconc                 @   i }| j                         D ]  \  }}t        |      dk  r||j                         k(  r(||j                         k7  s<|||j                         <   P||j                         j                         k(  sr|||j                         <    i || S )Nr!   )r   r:   r"   
capitalize)dekr   s       r   grow_dictionaryzLexicon.grow_dictionary|   s     GGIDAq1vzAGGI~&()Aalln%aggi**,, !'')  !zqzr'   c                 z   || _         d| _        i | _        i | _        t        j
                  j                  t        |rdnd d      5 }t        j                  t        j                  |            | _        d d d        t        j
                  j                  t        |rdnd d      5 }t        j                  t        j                  |            | _        d d d        t        d | j                  j                         D              sJ |rt        nt        | j                  j                         D ]l  }t!        |t"              rt        fd|D              r(J |       d|v sJ |       |j                         D ]   }|t        fd	|D              rJ |        n y # 1 sw Y   /xY w# 1 sw Y   xY w)
N)r   r!   gbusz
_gold.jsonz_silver.jsonc              3   `   K   | ]&  }t        |t              xs t        |t               ( y wr   )
isinstancestrdict)r   r   s     r   r   z#Lexicon.__init__.<locals>.<genexpr>   s)     ZFY:a%<At)<<FYs   ,.c              3   &   K   | ]  }|v  
 y wr   rN   r   r$   vocabs     r   r   z#Lexicon.__init__.<locals>.<genexpr>   s     2r!1:rr   DEFAULTc              3   &   K   | ]  }|v  
 y wr   rN   r   s     r   r   z#Lexicon.__init__.<locals>.<genexpr>   s     +B1AJr   )britishcap_stressesgoldssilvers	importlib	resources	open_textr   r   r   jsonloadr   valuesGB_VOCABUS_VOCABr   r   )selfr   rvsr   r   s        @r   __init__zLexicon.__init__   s]   $
  **4GD3Nj1YZ^_ 001>DJ [  **4GD3Nl1[\`a"22499Q<@DL ]ZdjjFWFWFYZZZZ#**##%B"c"2r226B62 B**A9+B+B(BEAEB % & [Z\\s   	.F$*.F1$F.1F:c                 4   |D cg c]<  }|j                         s| j                  j                  |j                               > }}d |v ryt	        dj                  |      d      }|j                  t        d      }t        j                  |      dfS c c}w )NNNr   r   r      )	isalphar   getupperr   r7   rsplitr   r   )r   rR   r$   rC   s       r   get_NNPzLexicon.get_NNP   s|    15EAdjjnnQWWY'E2:"''"+q)YY'+""2&)) Fs
   B+Bc                    |dk(  r#|t         v r| j                  t         |   d d|      S |t        v r| j                  t        |   d d |      S d|j                  d      v r]|j	                  dd      j                         r=t        t        |j                  d      t                    dk  r| j                  |      S |dv r|dk(  rd	dfS d
dfS |dv rL|j                  d      r| j                  |      S |j                  |dk7  s|r|dkD  r| j                  d   dfS y|dv r(|dk(  r"|j                  d      r| j                  |      S y|dk(  r|dk(  rt         ddfS |dv rt        j                  |      dk(  ry|dv s	|dk(  r&|dv r"| j                  d   ddd|j                     dfS |d v s
|d!k(  r%|d"k7  r |j                  |d!k7  rt         nd}|d#z   dfS |d$v s
|d%k(  r|dk(  r|j                  d&k(  rd'dfS d(dfS |d!k(  r*t#        j$                  d)|      r| j                  d*d d |      S |d+v r8|d,v r |j&                  r| j                  d-   d.   dfS | j                  d-   d/   dfS y0)1NADDr   rY   r   r(   r   )aADTu   ɐu   ˈA   )amAmAMNNr   r   )u   ɐmr   )anAnANr   )u   ɐnr   IPRP)byByBYADV)u   bˈIr   toToTOr   INr   u   təu   tʊ)NFT)inInr   NNPu   ɪn)theTheTHETu   ðiu   ðəz
(?i)vs\.?$versus)usedUsedUSED)VBDJJr   r   r   r   )ADD_SYMBOLSlookupSYMBOLSstripr   r   r:   r8   splitr   
startswithrG   r   r   r   get_parent_tagr   r   r   rH   )r   rR   r1   r,   ctxs        r   get_special_casezLexicon.get_special_case   s   %<DK/;;{40$cBBW_;;wt}dD#>>DJJsO#S"(=(E(E(GCPSTXT^T^_bTcilPmLnqrLr<<%%Z$;4144E144''~~d#||D))!!)TT\VQR
zz$'**''t|t 4||D))S[SE\&'q)1,,''G,B,B3,G5,P\!ddlsl7J**T*5FsGWGWXZ[[[\!ddlse|'*'7'7'?3$;^TVFE>1$$^##+,,45!CC&!CCD[RXXmT:;;xtS99--m#zz&)%0!33::f%i0!33r'   c                     | | S | j                  d      ry| j                  d      ry| j                  d      s| j                  d      ry| j                  d      s| j                  d      ry| S )	NVBVERBr   NOUNr   RBADJr   )r   )r1   s    r   r   zLexicon.get_parent_tag   sc    ;J^^D!^^D!^^E"cnnT&:^^E"cnnT&:
r'   c                 B   || j                   v s|t        v s|| j                  v ry|j                         rt	        d |D              syt        |      dk(  ry||j                         k(  r|j                         | j                   v ry|dd  |dd  j                         k(  S )NTc              3   >   K   | ]  }t        |      t        v   y wr   ordLEXICON_ORDSr#   s     r   r   z#Lexicon.is_known.<locals>.<genexpr>   s     *P4a3q6\+A4   Fr   )r   r   r   r   r   r:   r   r"   )r   rR   r1   s      r   is_knownzLexicon.is_known   s    4::DDLL4Hs*P4*P'PY!^TZZ\!djjldjj&@ABx48>>+++r'   c                    d }||j                         k(  r#|| j                  vr|j                         }|dk(  }| j                  j                  |      d}}||s| j                  j                  |      d}}t        |t              rC|r|j                  d|v rd}n||vrt        j                  |      }|j                  ||d         }|
|r"t        |vr| j                  |      \  }}|||fS t        ||      |fS )Nr   r   r   Noner   )r   r   r"   r   r   r   r   rG   r   r   r   r   r   )r   rR   r1   r,   r   is_NNPrC   r0   s           r   r   zLexicon.lookup   s    4::<D

$:::<DE\FZZ^^D)1F:f))$/Bb$s''/FbLB,,S1R	]+B:&^2%=d+JB~6z!B'//r'   c                 n    |sy |d   dv r|dz   S |d   dv r|| j                   rdz   dz   S dz   dz   S |dz   S )Nr   u   ptkfθr   u
   szʃʒʧʤ   ɪ   ᵻz)r   r   stems     r   _sz
Lexicon._s   sW    "X!#:"X%4<<4;cAAU;cAAczr'   c                 (   t        |      dk  s|j                  d      sy|j                  d      s| j                  |d d |      r|d d }n|j                  d      s0t        |      dkD  r=|j                  d      r,|j                  d	      s| j                  |d d
 |      r|d d
 }nAt        |      dkD  r2|j                  d	      r!| j                  |d d dz   |      r	|d d dz   }ny| j                  ||||      \  }}| j	                  |      |fS )Nr   r   r   ssr   'sr   esiesy)r:   endswithr  r   r  r   rR   r1   r,   r   r  r0   s          r   stem_szLexicon.stem_s  s   t9q=c 2}}T"t}}T#2Y'D9DmmD!c$i!md8KTXTaTabgThoso|o|  ~B  CF  DF  ~G  IL  pM9DY]t}}U3d3BiPSmUX8Y9s?D{{4fc:fwwt}f$$r'   c                     |sy |d   dv r|dz   S |d   dk(  r|| j                   rdz   dz   S dz   dz   S |d   dk7  r|dz   S | j                   st        |      dk  r|dz   S |d	   t        v r|d d d
z   S |dz   S )Nr   u
   pkfθʃsʧtr   r  r  r!   u   ɪdr  u   ɾᵻdu   ᵻdr   r:   US_TAUSr  s     r   _edzLexicon._ed  s    "X%#:"X_4<<4;cAAU;cAA"X_#:\\SY]%<"X 9x''f}r'   c                    t        |      dk  s|j                  d      sy|j                  d      s| j                  |d d |      r|d d }nLt        |      dkD  r=|j                  d      r,|j                  d      s| j                  |d d |      r|d d }ny| j                  ||||      \  }}| j	                  |      |fS )	Nr   r   r   ddr   edeedr  )r:   r  r  r   r"  r  s          r   stem_edzLexicon.stem_ed   s    t9q=c 2}}T"t}}T#2Y'D9DY]t}}T24==;OTXTaTabfgjhjbkmpTq9D{{4fc:fxx~v%%r'   c                     |sy | j                   r|d   dv ry |dz   S t        |      dkD  r|d   dk(  r|d   t        v r|d d dz   S |dz   S )Nr   u   əːr   r  r  u   ɾɪŋu   ɪŋr   r  s     r   _ingzLexicon._ing,  si     \\Bx6! f} Y]tBx348w3F9x''f}r'   c                    t        |      dk  s|j                  d      syt        |      dkD  r| j                  |d d |      r|d d }na| j                  |d d dz   |      r	|d d dz   }n@t        |      dkD  r1t        j                  d|      r| j                  |d d |      r|d d }ny| j                  ||||      \  }}| j                  |      |fS )N   ingr   r  r   z ([bcdgklmnprstvxz])\1ing$|cking$)r:   r  r  r   searchr   r)  r  s          r   stem_ingzLexicon.stem_ing;  s    t9q=e 4t9q=T]]49c:9D]]49S=#.9s?DY]ryy)LdSX\XeXefjknlnfoqtXu9D{{4fc:fyy&&r'   c                 L  
 | j                  |      \  }}|||fS |j                         
t        |      dkD  r|j                  dd      j	                         r||j                         k7  rdk7  st        |      dkD  r|| j
                  vr|| j                  vr||j                         k(  s|dd  |dd  j                         k(  rV
| j
                  v sF
| j                  v s8t        
fd| j                  | j                  | j                  fD              r
}| j                  |      r| j                  |      S |j                  d      r2| j                  |d d d	z         r| j                  |d d d	z         S |j                  d      r,| j                  |d d
       r| j                  |d d
       S | j                  |      \  }}|||fS | j                  |      \  }}|||fS | j                  |dn      \  }	}|	|	|fS y)Nr   'r   r      c              3   :   K   | ]  } |      d      yw)r   NrN   )r   fnr   r,   r1   wls     r   r   z#Lexicon.get_word.<locals>.<genexpr>S  s&      :6`2sFC(+6`s   zs'r  r  r   r   r   )r   r"   r:   r   r   r   r   r   anyr  r'  r/  r  r   r  )r   rR   r1   r,   r   rC   r0   r  r"  r)  r5  s     ```     @r   get_wordzLexicon.get_wordI  s   **4fcB
F>v:ZZ\t9q=T\\#r2::<AU5LCIM$**$T\\)ADJJL DHQR0@$@$**dll 2c :7;{{DLLRVR_R_6`: 7 D==s#;;tS&#66]]4 T]]49t3CS%I;;tCRy4/fcBB]]3DMM$s)S$A;;tCRy#vs;;[[sFC8
F>v:ll4fc:V?;}}T3v~6SVWf<r'   c                     d| vry| j                  d      dkD  ry| j                  d      d   }t        |      dk  xs t        |      dhk(  S )NrY   Tr   Fr   r   )countr   r:   set)rR   centss     r   is_currencyzLexicon.is_currencyi  sO    d?ZZ_q 

3"5zA~2Us!22r'   c           	      	    t        j                  d|      }|r|j                         nd }|r|d t        |        n|}g |j	                  d      r(j                   j                  dd d d              |dd  }d( fd	}t        |      r(|t        v r  |t        t        |      d	      d
       nsAt        |      dk(  r3|t        vr+t        |      r  |t        t        |      d	      d
       n@|sd|vr|j                  dd      }|d   dk(  st        |      dkD  r|D cg c]  } ||d       c} nt        |      dk(  re|j                  d      sT ||d          |d   dk(  r2j                   j                  dd dd               ||d   d       n ||dd  d       n ||       n{|j                  d      dkD  s|s}d}	|j                  dd      j                  d      D ]U  }|snN|d   dk(  s#t        |      dk7  r.t!        d |dd  D              r|D cg c]  } ||d       c} n
 |||	       d}	W n|t        v r1t"        j%                  |      rt'        |j                  dd      j                  d      t        |         D 
cg c]  \  }}
|rt        |      nd|
f }}}
t        |      dkD  r!|d   d   dk(  r|d d }n|d   d   dk(  r|dd  }t)        |      D ]  \  }\  }}
|dkD  r#j                   j                  dd d d               |||dk(         j                  t+        |      dk7  r|
dk7  r j-                  |
dz   d d d       n j                  |
d d d               nt        |      rt        t        |      d	      }nd|vr1t        t        |j                  dd            |t        v rdnd	      }nM|j                  dd      }|d   dk(  rddj/                  d |dd  D              z   }nt        t1        |            } ||d
       st3        d d!||       y"dj/                  d# D              t5        d$ D              c}|d%v r j7                        |fS |d&v r j9                        |fS |d'k(  r j;                        |fS |fS c c}w c c}w c c}
}w ))Nz[a-z']+$-minusr   TFc           	         t        j                  d|r| nt        t        |                   }t	        |      D ]  \  }}|dk7  sdv r[|r.|dk(  r)t        |      dkD  r|dk(  rdv rj                  d       ?j                  j                  |d |d	k(  rd
nd d              j|dk(  spdv susxd   d   dz   d   d   fd<    y )Nz[^a-z]+rz   r~   r   r   oner   )u   ər   pointr  nr   u   ən)r   r   r   intr   r:   appendr   )	numfirstescapesplitsr   wr.   resultr   s	         r   
extend_numz&Lexicon.get_number.<locals>.extend_numz  s    XXj#Ys3x=PQF!&)1:	!1aCK!OU
sV_O_i0dkk!Tg2SWY]&^_%ZC9$4"(*Q-%"7A!GF2J *r'   ordinal)r   )rH  r   yearrY   rZ   r   r   0r   )rG  00Or  r!   c              3   &   K   | ]	  }|d k7    yw)rO  NrN   r   rC  s     r   r   z%Lexicon.get_number.<locals>.<genexpr>  s     <Ww!Q#Xw   rz   ro   r   cardinalzpoint r   c              3   D   K   | ]  }t        t        |              y wr   )r   rD  rS  s     r   r   z%Lexicon.get_number.<locals>.<genexpr>  s     .S(QyQ/@(s    u   ❌zTODO:NUMr   c              3   &   K   | ]	  \  }}|  y wr   rN   )r   rX   r5   s      r   r   z%Lexicon.get_number.<locals>.<genexpr>  s     !71!rT  c              3   &   K   | ]	  \  }}|  y wr   rN   )r   r5   r   s      r   r   z%Lexicon.get_number.<locals>.<genexpr>  s     <R641aQ6rT  )r   r  )r%  'dr,  )TF)r   r.  groupr:   r   rE  r   r   ORDINALSr   rD  
CURRENCIESr   r  r9  r   r6  r   r<  zipr   absr  r7   floatprintr=   r  r"  r)  )r   rR   r-   r*   r.   suffixrL  rF  rC  rG  unitpairsr   r0   rK  s   `   `         @r   
get_numberzLexicon.get_numberr  s   ;-#)t&,tMc&k\"$??3MM$++gtT4@A8D		H D>f0yTy9$GCINxz/IhW[nyTv6tDS_,,sB'C1v}C1589SAU+S9SQs||D'93q6"q6S=MM$++c4T"BCs1vU3s12we43ZZ_q E||C,2237Vs]s3x1}<WsSTSUw<W9W9<=AZ/=s%0 8 #(;(;D(AEHVY[]I^IdIdehIikuv~k  FA  B  FA	T#c#h1d3  FAE  B5zA~8A;!#!"1IE1Xa[A%!!"IE"+E"2;Cq5MM$++eT4"FG3ad+SUV[_cj[jdkk$s(D$Eptp{p{  }A  CG  IM  OS  qT  U	 #3 ~ Tz:D T\\#r%:!;VW_M_	eop||C,7c>#chh.S$qr(.S&SSD$U4[1DtD)%T84!7!77<R6<R9R[ 776?F**|#88F#V++u_99V$f,,v~m :  >
 Bs   ,S -S&S
c                     |s|S t         j                  |      }|r| j                  |d   dz   d d d       d   nd }|r| d| S |S )Nr   r   r   )r\  r   r  )r   rC   r-   s      r   append_currencyzLexicon.append_currency  sV    I>>(+HP4;;x{3dDA!DVZ%-"Qxj!525r'   c                     | j                         s| S t        j                  |       }|t        |      k(  rt	        t        |            S | S r   )isdigitunicodedatanumericrD  r   )r$   rC  s     r   numeric_if_neededzLexicon.numeric_if_needed  s=    yy{H"3q6ks3q6{0q0r'   c                     t        d | D              ryddddgt        d}|D ]$  }| j                  |      s| d t        |        }  n t        fdt	        |       D              S )	Nc              3   4   K   | ]  }t        |         y wr   r   r#   s     r   r   z$Lexicon.is_number.<locals>.<genexpr>  s     -18A;rB   Fr,  rY  r%  r  r   c              3   j   K   | ]*  \  }}t        |      xs |d v xs xr |dk(  xr |dk(   , yw)z,.r   r>  Nrn  )r   r   r$   r*   s      r   r   z$Lexicon.is_number.<locals>.<genexpr>  sB     m]lUYUVXY8A;P!t)P0OAF0OqCxP]ls   03)r   r[  r  r:   r   )rR   r*   suffixesr   s    `  r   	is_numberzLexicon.is_number  sp    ---4t<h<<A}}QHc!fW~  m]fgk]lmmmr'   c                    |j                   j                  |j                  n|j                   j                  j                  t	        d      d      j                  t	        d      d      }t        j                  d|      }dj                  d |D              }||j                         k(  rd n(| j                  t        ||j                         k(           }| j                  ||j                  ||      \  }}|Ft        | j                  ||j                   j                         |j                   j"                        |fS t$        j'                  ||j                   j(                        ru| j+                  ||j                   j                   |j                   j(                  |j                   j,                        \  }}t        ||j                   j"                        |fS t/        d |D              syy)	Ni   r1  i   NFKCr   c              3   F   K   | ]  }t         j                  |        y wr   )r   rk  r#   s     r   r   z#Lexicon.__call__.<locals>.<genexpr>  s     BTw003Ts   !c              3   >   K   | ]  }t        |      t        v   y wr   r  r#   s     r   r   z#Lexicon.__call__.<locals>.<genexpr>  s     :TSV|+Tr  r   )r5   r+   r   r   chrri  	normalizer7   r"   r   rD  r   r7  r1   r   rf  r-   r,   r   rq  r*   rd  r.   r   )r   r   r   rR   r,   rC   r0   s          r   __call__zLexicon.__call__  sk   44::-244::FFs4yRUV^^_bcg_hjmn$$VT2wwBTBB-43D3DSQUQ[Q[Q]I]E^3_]]4=
F> 4 4R GUW]]]tRTT\\2rtt}}bddllBDDNN[JBBDDKK0&88:T::
 r'   N)rI   rJ   rK   staticmethodr   r   r   r   r   r  r   r  r  r"  r'  r)  r/  r7  r<  rd  rf  rk  rq  rx  rN   r'   r   r   r   {   s     F&*$L  	,0(% 
&'@ 3 3Ob6 1 1 n nr'   r   c            	           e Zd ZddZed        Zdedee   fdZ	dee   dee   fdZ
edee   deeeee   f      fd	       Zed
        Zed        Zddedeeee   f   fdZy)G2PNc                 :   || _         || _        d|rdnd }t        j                  j	                  |      st        j
                  j                  |       |rdnddg}t        j                  ||      | _        t        |      | _
        |r|nd | _        || _        y )Nen_core_web_trfsmtransformertok2vectagger)enable)versionr   spacyutil
is_packageclidownloadr   nlpr   lexiconfallbackr   )r   r  r~  r   r  r   name
componentss           r   r   zG2P.__init__  s    se56zz$$T*IIt$'*m	8D
::d:6w'$,$r'   c                    d}g }i }d}| j                         } t        j                  |       D ]T  }|| ||j                          z  }|j	                  | ||j                          j                                |j                  d      }t        ||d d dv rdndd        rt        |      }n|dv rd}n||dk(  rd	}ntt        |      dkD  r+|d   d
k(  r#|d   d
k(  r|d   |dd  j                  d
      z   }n;t        |      dkD  r+|d   dk(  r#|d   dk(  r|d   |dd  j                  d      z   }nd }|||t        |      <   ||j                  d      z  }|j                  |j                  d             |j                         }W |t        |       k  r*|| |d  z  }|j	                  | |d  j                                |||fS )Nr   r   r!   r   )r>  r   )z0.5z+0.5r   z-0.5r   rx   r   rb   )lstrip
LINK_REGEXfinditerstartextendr   rZ  r   rD  r:   rstriprE  end)r   rK  r   featureslast_endmfs          r   
preprocesszG2P.preprocess  s   {{}$$T*Ad8AGGI..FMM$x	288:;
Aq!u
2!:;<Fo%fQ!!"aD1QR5<<,,Q!!"aD1QR5<<,,}()V%aggaj FMM!''!*%uuwH) +* c$id89o%FMM$xy///12vx''r'   r   r   c                    | j                  |      }|D cg c]E  }t        |j                  |j                  |j                  t        j
                  ddd            G }}|s|S t        j                  j                  j                  ||D cg c]  }|j                   c}      }|j                         D ]9  \  }}	t        |	t              st        |	t              s|	dv s	J ||	f       t        t        j                   |j"                  j$                  |k(        d         D ]  \  }
}|t'        |      k\  rt        |	t              s|	||   j(                  _        :|	j-                  d      rL|
dk(  ||   j(                  _        |
dk(  r|	j1                  d      nd||   _        d	||   j(                  _        |	j-                  d
      s|	j1                  d
      ||   j(                  _         < |S c c}w c c}w )NTr   F)r*   r.   r/   )r   r1   r   r5   )r   r   r   rx   r+  rb   )r  r   r   tag_whitespace_r9   r  training	Alignmentfrom_stringsr   r   r   rD  r   npwherey2xr   r:   r5   r,   r   r*   r  r2   r0   r.   )r   r   r   r  docr   mutable_tokensalignr   r   r   r   s               r   tokenizezG2P.tokenize  s   hhtn
  b !bgg"..UK
  	  !!((55fQ_>`Q_2rwwQ_>`aNN$DAqa%As);qK?OWRSUVQWWO!"((599>>Q+>"?"BC1N++!!S)12N1%''.\\#&23q&N1%''/BCq&#bN1%.12N1%''. \\#&45HHSMN1%''1 D % - ?as   A
G7G<
r   c                     g }|D ]T  }|r=|j                   j                  s't        |j                         |g| j                        n|}|j                  |       V |S )Nr   )r5   r*   r>   popr   rE  )r   r   rK  r   s       r   	fold_leftzG2P.fold_left8  sS    BCIRTRVRVR^R^vzz|R0dhh?dfBMM"  r'   c                 \   g }d }t        |       D ]]  \  }}|j                  j                  {|j                  ot	        |j
                        D cg c]P  }t        ||dt        j                  d|j                  j                  |j                  j                  d            R }}n|g}|j                  |d   _        t        |      D ]  \  }}|j                  j                  |j                  n|j                  dk(  r8|j
                  t        v r&|j
                  }d|_        d|j                  _        n|j                  d	k(  r(|j
                  d
v rd|_        d|j                  _        nU|j                  t        v rst!        d |j
                  D              sWt"        j%                  |j                  dj'                  d |j
                  D                    |_        d|j                  _        n|[|j                  dk7  rd }n|dz   t)        |      k(  r|dz   t)        |       k(  s| |dz      j                  dk7  r||j                  _        nsd|cxk  rt)        |      dz
  k  r\n nY|j
                  dk(  rJ||dz
     j
                  d   ||dz      j
                  d   z   j-                         rd|j                  _        |j                  j                  |j                  |j/                  |       3|rLt1        |d   t2              r9|d   d   j                  s'd|j                  _        |d   j/                  |       |j/                  |j                  r|n|g        ` |D 	cg c]'  }	t1        |	t2              rt)        |	      dk(  r|	d   n|	) c}	S c c}w c c}	w )Nr   TF)r*   r.   r,   r/   )r   r   r5   r   ra   r   r`   )r>  u   –u   —r   c              3   j   K   | ]+  }d t        |j                               cxk  xr dk  nc  - yw)rj   z   N)r  r"   r#   s     r   r   z!G2P.retokenize.<locals>.<genexpr>V  s+     5c[bVWbC	N6Qc6Q6Q[bs   13c              3   2   K   | ]  }|t         v s|  y wr   )PUNCTSr#   s     r   r   z!G2P.retokenize.<locals>.<genexpr>W  s     HkT[q_`dj_jT[s   CDr   r   2r   )r   r5   r+   r2   subtokenizer   r   r   r9   r.   r,   r   r1   r\  r0   
PUNCT_TAGSr   PUNCT_TAG_PHONEMESr   r7   r:   r-   r   rE  r   r;   r*   )
r   wordsr-   r   tokenr  tksr   r   rJ  s
             r   
retokenizezG2P.retokenize?  s   !&)HAuww}}$)? 'uzz24 3a b''@Q@QZ_ZaZaZhZhsxy 3  4
 g!&!1!1CG"3244::)R[[-DVVs]rww*'<!wwH"$BK"#BDDKVVs]rww,'>"'BK"#BDDKVVz)#5c[][b[b5c2c"4"8"8HkTVT[T[HkAk"lBK"#BDDK )vv~#'1CacS[.@F1Q3KOOW[D[(0'SXaZ'BGGsNAaCb@QRUVWXYVYRZR_R_`aRb@b?k?k?m!%BDDJ44::)R[[-DLL$z%)T:59R=C[C[#(BDDL"I$$R(LLr}}2$?9 ( *L MRREq
1d+A!!BERRI4H Ss   AN$5,N)c                     | j                   }|rt        d |D        |      n|}|j                  dv xs |j                  dk(  xr |j                  dv }t	        ||      S )Nc              3      K   | ]:  t        fd t        t        t        fD              s't        v rdnt        v  < yw)c              3   &   K   | ]  }|v  
 y wr   rN   )r   r   r$   s     r   r   z.G2P.token_context.<locals>.<genexpr>.<genexpr>n  s(       ZR  kQefZ[_`Z`  kQr   N)r6  r   
CONSONANTSNON_QUOTE_PUNCTSr#   s    @r   r   z$G2P.token_context.<locals>.<genexpr>n  sT       SPR1VY  ZRkqs}  @P  kQ  ZR  WRa#33d!v+FPRs
   (AAr   r   r   )rG   rH   )rG   r   r   r1   rF   )r   rC   r  vowelrH   s        r   token_contextzG2P.token_contextk  ss       _a  SPR  S  UZ  [  glJJ,.d5::3E3c%))WcJc	)DDr'   c                    dj                  d | d d D              | d   j                  z   }d|v xsJ d|v xsD t        |D ch c],  }|t        vs|j	                         rdnt        |      rdnd. c}      dkD  }t        |       D ]  \  }}|j                  ||t        |       dz
  k(  r5|j                  t        v r#|j                  |_        d	|j                  _
        Xt        d
 |j                  D              sud|_        d	|j                  _
        |dkD  s||j                  _         |ry t        |       D cg c]:  \  }}|j                  st        |j                  v t        |j                        |f< }}}t        |      dk(  rKt        | |d   d      j                        dk(  r*|d   d   }t        | |   j                  d      | |   _        y t        |      dk  s$t!        d |D              t        |      dz   dz  k  ry t#        |      d t        |      dz   }|D ]'  \  }}}t        | |   j                  d      | |   _        ) y c c}w c c}}w )Nr   c              3   N   K   | ]  }|j                   |j                  z     y wr   r   r   s     r   r   z%G2P.resolve_tokens.<locals>.<genexpr>t  s     E2rww.r   r   r   rx   r   r   r!   r   c              3   ,   K   | ]  }|t         v   y wr   SUBTOKEN_JUNKSr#   s     r   r   z%G2P.resolve_tokens.<locals>.<genexpr>{  s     >gn,g   r   c              3   (   K   | ]
  \  }}}|  y wr   rN   )r   br5   s      r   r   z%G2P.resolve_tokens.<locals>.<genexpr>  s     $>g71aQgs   )r7   r   r:   r  r   r   r   r2   r  r5   r0   r   r/   r   rD   r   r%   r<   )r   r   r$   r/   r   r   indicesr5   s           r   resolve_tokenszG2P.resolve_tokensr  sP   wwEEEr
W$;  U#+  Uos  6Posjkwx  AO  xO199;aRZ[\R]Qcd6eos  6P  2Q  TU  2Uv&EAr{{"Fa'BGG7G,G"$''BK"#BDDK>bgg>>"$BK"#BDDKQ ( ' _hio_p  A_pV[VWY[tvttNbkk1=3MqQ_p  Aw<1VGAJqM%:%?%?!@A!E
1A!-fQi.@.@$!GF1I\A$>g$>!>3w<PQ>VWBW!W/"23w<?3GAq!!-fQi.@.@$!GF1I + 6P As   I#I?I*Ic                 z    |dk(  rt         j                  n|}|r ||      n|g i f\  }}} j                  |||      } j                  |      }t         j	                  |      }t               }t        t        t        |                  D ]  \  }}t        |t              s|j                  4 j                  t        ||j                        |      \  |_
        |_        |j                  ? j                  3 j                  t        ||j                              \  |_
        |_        t         j!                  ||j                  |      }dt#        |      }	}d}
||	k  rt%        d |||	 D              rd }nt'        |||	       }|dn j                  ||      \  }}|U|||   _
        |||   j                  _        ||dz   |	 D ]  }d|_
        ||_         t         j!                  |||      }|}	d}nj|dz   |	k  r|dz  }n\|	dz  }	||	   }|j                  Dt)        d	 |j*                  D              rd|_
        d
|j                  _        n j                  d}
n	d}||	k  r|
rt'        |      } j                  |      \  |d   _
        |d   j                  _        t-        dt#        |            D ]7  }d||   _
        |d   j                  j                  ||   j                  _        9 lt         j/                  |        |D cg c]+  }t        |t              rt'        | j0                        n|- }} j2                  dk7  rE|D ]@  }|j                  s|j                  j                  dd      j                  dd      |_
        B dj5                   fd|D              }||fS c c}w )NT)r5   r   Fc              3   l   K   | ],  }|j                   j                  d uxs |j                  d u . y wr   )r5   r+   r2   r   s     r   r   zG2P.__call__.<locals>.<genexpr>  s1     aS`Rrttzz-HD1HHS`s   24r   r   r   c              3   ,   K   | ]  }|t         v   y wr   r  r#   s     r   r   zG2P.__call__.<locals>.<genexpr>  s     DGqqN2Gr  r   r  z2.0u   ɾTu   ʔr  c              3      K   | ]5  }|j                   j                  n|j                   |j                  z    7 y wr   )r2   r   r   )r   r   r   s     r   r   zG2P.__call__.<locals>.<genexpr>  s1     mfl`bbkk&9$((r{{bmm[fls   ;>)r{  r  r  r  r  rF   reversedr;   r   r   r2   r  r   r5   r0   r  r  r:   r6  r>   r   r   ranger  r   r  r7   )r   r   r  r   r  r   r   rJ  leftrightshould_fallbackr   rC   r0   xr   rK  s   `                r   rx  zG2P.__call__  s^   '1T'9S^^z
5?D!1dBPR^fhtVX6''nT)F"345DAqa&::%+/<<QSS8I3+O(AJ::%$--*C+/==acc9J+K(AJ''QZZ;SV%D#O,aSTUYZ_S`aaB%aUm4B-/Z\T\\"c=R
F>')AdG$'-AdGII$tAve_%'
#) - ++CR8C EDAX%AIDQJE5B{{*DBGGDD*,BK*+BDDK!]]6.2O!D7 ,8 !!_/3}}R/@,!qtvv}q#a&)A$&AaDM$%aDFFMMAaDFFM * ""1%[ 6\ \bb[aUWJr44H,rtxx0bP[ab<<5 ;;"$++"5"5dC"@"H"Hs"SBK  mflmmv~ cs   0N8)NFFNu   ❓)T)rI   rJ   rK   r   ry  r  r   r	   r   r  r  r   r  r  r  r   rx  rN   r'   r   r{  r{    s    
 ( (@S tF| 6V f  )S4< )SDvtF|7K1L,M )S )SV E E H H4;S ;eCf<M6N ;r'   r{  r   )7r   r   r  r   dataclassesr   r   r   typingr	   r
   r   r   importlib.resourcesr   r   numpyr  r   r  ri  r   r>   	frozensetrA   rD   rF   rV   r  rU   r  r  r  r  r  rv  r  r  r  r  r!  r\  r[  r   r   r   r   r   r   r   r   r   r   r   r{  rN   r'   r   <module>r     s     *  / /    	  f HSM V < |$
E   
> $% RZZ34
>*	(	)EEE  QR
!3SYSYUXY]U^_ B8r28r38<=

-
. 



 -.g&eT
:_`[\!A; 	5	66-r rhY Yr'   