
    -cJj                         d dl Z d dlmZ d dlmZmZmZ 	 	 	 	 ddee   dededee   dee   d	ed
eeeee	f         fdZ
y)    N)Path)ListOptionalUniontextstokenstokens_type	bpe_modellexicon
output_idsreturnc                    	 ddl }	 ddlm ddlmm t        |      j                         s
J d	|        i }t        |d
d      5 }|D ]d  }	|	j                         j                         }
t        |
      dk(  sJ t        |
             |
d   |vsJ d|
 d       t        |
d         ||
d   <   f 	 ddd       d|v rDt        |      j                         s
J d	|        |j                         }|j!                  |       i }|dk(  r|rt        |      j                         s
J d	|        t        |d
d      5 }|D ]M  }	|	j                         j                         }
t        |
      dk\  sJ t        |
             |
d   }|
dd }|||<   O 	 ddd       g }dt"        dt"        dt$        t"           ffd}|dk(  r7| D cg c]*  }t'        dj)                  |j                                     , }}n|dk(  rj+                  | t"              }n|dk(  s|dk(  r!| D ]  }|j-                   |||              nX|dk(  rt/        j0                  d      }| D ]  }|j                         j                         }g }d}|D ]B  }||v r	|||   z  }|j3                  |      t        d| d| d        d!} n| ||d      z  }D |ro|j-                  |        n|d"k(  s
J d#|        t/        j0                  d$      }| D ]  }|j                  |      }|D cg c]!  }t        |j                               dkD  s |# }}g }|D ]9  }|j3                  |      |j-                  |       &|j5                  |      z  }; |j-                  |        g }|D ]V  }g }d}|D ]7  }||v r|j-                  |r||   n|        t        d%| d&| d'| d        d!} n |rF|j-                  |       X |S # t        $ r# t        d       t        d       t        d        w xY w# t        $ r# t        d       t        d       t        d        w xY w# 1 sw Y   xY w# 1 sw Y   xY wc c}w c c}w )(a  
    Encode the given texts (a list of string) to a list of a list of tokens.

    Args:
      texts:
        The given contexts list (a list of string).
      tokens:
        The path of the tokens.txt.
      tokens_type:
        The valid values are cjkchar, bpe, cjkchar+bpe, fpinyin, ppinyin, phone+ppinyin.
        fpinyin means full pinyin, each cjkchar has a pinyin(with tone).
        ppinyin means partial pinyin, it splits pinyin into initial and final,
        phone means English phonemes in CMU dictionary format.
      bpe_model:
        The path of the bpe model. Only required when tokens_type is bpe or
        cjkchar+bpe.
      lexicon:
        The path of the lexicon.txt. Only required when tokens_type is phone+ppinyin.
      output_ids:
        True to output token ids otherwise tokens.
    Returns:
      Return the encoded texts, it is a list of a list of token ids if output_ids
      is True, or it is a list of list of tokens.
    r   Nz
Please runz  pip install sentencepiecezbefore you continue)pinyin)to_initialsto_finals_tonez  pip install pypinyinzFile not exists, rzutf-8)encoding   zDuplicate token:     bpezphone+ppinyintxtout_typer   c                 ,   |dv s
J d|         |       D cg c]  }|d   	 }}d|k(  rbg }|D ]Y  } 	|d      } |d      }|dk(  r|dk(  r|j                  |       3|r|j                  |       |sI|j                  |       [ |S |S c c}w )N)ppinyinfpinyinzgiven r   r   F)strict )append)
r   r   xpyresinitialfinalr   r   r   s
          [/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/sherpa_onnx/utils.py	to_pinyinztext2token.<locals>.to_pinyinV   s    11FVH:3FF1"3K(KqadK( C%a6&q7b=Ub[JJqM

7+

5)  JI )s   Bcjkcharr   )r   r   r   z^[\u4e00-\u9fff]+$FzWord z> not in lexicon and it is not a CJK character, skipping text: .Tzcjkchar+bpez[Supported tokens_type are cjkchar, bpe, cjkchar+bpe, ppinyin, fpinyin, phone+ppinyin given z([\u4e00-\u9fff])zCan't find token z. in token table, check your tokens.txt see if z in it. skipping text : )sentencepieceImportErrorprintpypinyinr   pypinyin.contrib.tone_convertr   r   r   is_fileopenstripsplitlenintSentencePieceProcessorloadstrr   listjoinencoder   recompile	fullmatchencode_as_pieces) r   r   r	   r
   r   r   spmtokens_tableflinetoksspphone_tablewordphones
texts_listr&   textr   patternwords	text_list	skip_textwchars	mix_charsch_or_wresultcontain_oovr   r   r   s                                 @@@r%   
text2tokenrS      s   @##M <!?%6vh#??!L	fcG	,D::<%%'Dt9>,3t9,>7,.K2CD60KK.$'QLLa!	  
- I&&(I,=i[*II('')
	Ko%W--/	)wi(	)/'31Qzz|))+4yA~0s4y0~Awab$*D!  2 #%Js c d3i & i>CDedd2774::<01e
D		YYusY3
			![I%=Ci[9: 		' **23DJJL&&(EII#Q/I((+3#A3 '..2V16 %)	!Yq)%<<	  !!),% * =(	whithuv	w(
 **12D MM$'E$)@EqS^a-?EI@I$$$W-9$$W- !4!4W!==I % i( " +-F	Cl"  jc!2cJ'u -)),-EdV1N #  MM)$! " MQ  l+,#$	  l&'#$	 
-	," 21< ET AsB   O; P* A*Q AQ&/Q3!!Q8Q8;,P'*,QQ#&Q0)r'   NNF)r:   pathlibr   typingr   r   r   r6   boolr3   rS        r%   <module>rY      s    	  ( ( !#!j9jj j }	j
 c]j j 
$uS#X
 jrX   