
    (HJj                        d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
mZmZmZ d dlmZ d dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ  ej@                  ejB                  d	
        ejD                  e#      Z$e G d d             Z%e G d d             Z&e G d d             Z' G d d      Z( G d d      Z) G d d      Z* G d d      Z+ G d d      Z, G d d      Z- G d d      Z. G d d       Z/ G d! d"      Z0 G d# d$      Z1d% Z2e#d&k(  r	  ejf                   e2              yy# e4$ r Y yw xY w)'    N)ThreadPoolExecutor)	dataclass)AnyCallableIterableOptional)generateload)AudioPlayer)
load_modelz4%(asctime)s - %(name)s - %(levelname)s - %(message)s)levelformatc                   (   e Zd ZU dZdZeed<   dZee   ed<   dZ	eed<   dZ
eed	<   d
Zeed<   dZeed<   dZee   ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   d Zeed!<   d"Zeed#<   d$Zeed%<   d&Zeed'<   d(Zeed)<   d*Zeed+<   d,Zeed-<   d.Z eed/<   dZ!ee   ed0<   dZ"ee   ed1<   d2Z#e$ed3<   d4Z%eed5<   d6Z&eed7<   d Z'eed8<   d9Z(eed:<   dZ)eed;<   d<Z*eed=<   d2Z+e$ed><   d?Z,eed@<   dAZ-e$edB<   dC Z.y)DVoicePipelineConfigzCConfiguration for the realtime Voxtral STT + Pocket TTS voice loop.>  input_sample_rateNoutput_sample_rate   input_channels    frame_duration_msbalancedlatency_profile0mlx-community/Voxtral-Mini-4B-Realtime-2602-4bit	stt_modelstt_transcription_delay_ms   stt_max_decode_tokens_per_step   stt_max_turn_tokens`   stt_finalization_max_stepsmlx-community/silero-vad	vad_modelffffff?vad_start_threshold皙?vad_stop_thresholdvad_start_framesX  vad_end_silence_msg      >@vad_max_turn_seconds   
preroll_msmlx-community/smart-turn-v3
turn_model      ?turn_thresholdi@  turn_max_incomplete_silence_ms1mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-4bitresponse_modelzlYou are a helpful voice assistant. Respond in natural spoken sentences. Never use markdown, emoji, or lists.system_promptmlx-community/pocket-tts	tts_modelcosette	tts_voicetts_streaming_intervaltts_temperatureTbarge_in   min_barge_in_ms  ignore_playback_echo_msecho_delay_min_ms  echo_delay_max_msecho_correlation_step_ms   barge_in_min_transcript_chars
play_audio   
queue_sizeFverbosec                     | j                   j                         }| j                  ddddj                  |d      | _        | j                  ddddj                  |d      | _        y y )N     i  fastr   qualitygQ?{Gz?gQ?)r   lowerr   getr<   )selfprofiles     f/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_audio/sts/voice_pipeline.py__post_init__z!VoicePipelineConfig.__post_init__K   s}    &&,,.**2/ c'3	 + &&. + c'4 	 ' /    )/__name__
__module____qualname____doc__r   int__annotations__r   r   r   r   r   strr   r   r   r!   r#   r%   r'   floatr)   r*   r,   r-   r/   r1   r3   r4   r6   r7   r9   r;   r<   r=   r>   boolr@   rB   rC   rE   rF   rH   rI   rK   rL   rY    rZ   rX   r   r      s   M#s#(,,NCs%OS%GIsG044*+"C+""&((/Is/!%% ##c!!"&%&J3J3NE*."C.MNCMv 3  0Is/Is.2HUO2'+OXe_+HdOS#&S& s  s $&c&)*!3*JJGT!rZ   r   c                   >    e Zd ZU eed<   eed<   dZeed<   dZeed<   y)SpeechFrameDecisionprobability	is_speechFspeech_startedcandidate_endedN)r[   r\   r]   rb   r`   rc   ri   rj   rd   rZ   rX   rf   rf   \   s!    O ND !OT!rZ   rf   c                   "    e Zd ZU eed<   eed<   y)EndpointDecisioncompleterg   N)r[   r\   r]   rc   r`   rb   rd   rZ   rX   rl   rl   d   s    NrZ   rl   c                   Z    e Zd ZdZd Zdeg ef   defdZdeg ef   defdZd
dZ	d
d	Z
y)MLXWorkSchedulera`  Run all MLX work on one worker thread and stream.

    Recent MLX streams are thread-local, so model load, cached state creation,
    and inference must stay on the same worker thread. A generic
    ``asyncio.to_thread`` call can hop between pool threads and later fail when
    evaluating arrays that reference a stream created in another thread.
    c                 D    t        dd      | _        d | _        d| _        y )Nr   zmlx-audio-sts)max_workersthread_name_prefixF)r   	_executor_stream_initializedrV   s    rX   __init__zMLXWorkScheduler.__init__s   s$    +o
 !rZ   funcreturnc                 .   | j                   sNt        j                  t        j                        | _        t        j
                  | j                         d| _         t        j                  | j                        5   |       cd d d        S # 1 sw Y   y xY w)NT)ru   mx
new_streamgpurt   set_default_streamstream)rV   rx   s     rX   _runzMLXWorkScheduler._runz   sZ      ==0DL!!$,,/ $DYYt||$6 %$$s   :BBc                    K   t        j                         }|j                  | j                  | j                  |       d {   S 7 wN)asyncioget_running_looprun_in_executorrs   r   )rV   rx   loops      rX   runzMLXWorkScheduler.run   s7     '')))$..$))TJJJJs   ?AAANc                 \    | j                    t        j                  | j                          y y r   )rt   r{   synchronizerv   s    rX   _synchronizezMLXWorkScheduler._synchronize   s!    <<#NN4<<( $rZ   c                    K   t        j                         }|j                  | j                  | j                         d {    | j                  j                  dd       y 7 "w)NTF)waitcancel_futures)r   r   r   rs   r   shutdown)rV   r   s     rX   r   zMLXWorkScheduler.shutdown   sQ     '')""4>>43D3DEEET%@ 	Fs   >A% A##A%ry   N)r[   r\   r]   r^   rw   r   r   r   r   r   r   rd   rZ   rX   ro   ro   j   sR    ""c'* s Khr3w/ KC K)ArZ   ro   c                   V    e Zd ZdefdZdej                  deej                     fdZy)FixedSizeAudioChunker
chunk_sizec                 x    t        |      | _        t        j                  dt        j                        | _        y )Nr   dtype)r_   r   npzerosfloat32_buffer)rV   r   s     rX   rw   zFixedSizeAudioChunker.__init__   s#    j/xx4rZ   samplesry   c                 "   t        j                  |t         j                        j                  d      }|j                  dk(  rg S t        j
                  | j                  |g      | _        g }| j                  j                  d   | j                  k\  r{|j                  | j                  d | j                   j                                | j                  | j                  d  | _        | j                  j                  d   | j                  k\  r{|S )Nr   r   )r   asarrayr   reshapesizeconcatenater   shaper   appendcopy)rV   r   chunkss      rX   pushzFixedSizeAudioChunker.push   s    **WBJJ7??C<<1I~~t||W&=>ll  #t6MM$,,'89>>@A<<(9:DL ll  #t6 rZ   N)	r[   r\   r]   r_   rw   r   ndarraylistr   rd   rZ   rX   r   r      s.    53 5
BJJ 
4

+; 
rZ   r   c                   d    e Zd ZdefdZdej                  ddfdZdej                  fdZd	dZ	y)
PreRollBuffermax_samplesc                 T    t        dt        |            | _        g | _        d| _        y Nr   )maxr_   r   _chunks_samples)rV   r   s     rX   rw   zPreRollBuffer.__init__   s$    q#k"23)+rZ   r   ry   Nc                    | j                   dk  ry t        j                  |t        j                        j	                  d      }|j
                  dk(  ry | j                  j                  |j                                | xj                  |j
                  z  c_	        | j                  | j                   kD  r| j                  r| j                  | j                   z
  }| j                  d   }||j
                  k\  r;| j                  j                  d       | xj                  |j
                  z  c_	        n(||d  | j                  d<   | xj                  |z  c_	        y | j                  | j                   kD  r| j                  ry y y y )Nr   r   r   )r   r   r   r   r   r   r   r   r   r   pop)rV   r   extrafirsts       rX   r   zPreRollBuffer.append   s   q **WBJJ7??C<<1GLLN+%mmd...4<<MMD$4$44ELLOE

"  #+"'-Q& mmd...4<<.<.rZ   c                     | j                   s%t        j                  dt        j                        S t        j                  | j                         j                  t        j                  d      S )Nr   r   Fr   )r   r   r   r   r   astyperv   s    rX   rU   zPreRollBuffer.get   sE    ||88ARZZ00~~dll+222::E2JJrZ   c                 F    | j                   j                          d| _        y r   )r   clearr   rv   s    rX   r   zPreRollBuffer.clear   s    rZ   r   )
r[   r\   r]   r_   rw   r   r   r   rU   r   rd   rZ   rX   r   r      s<    C 
bjj T &KRZZ K
rZ   r   c                       e Zd ZdZddddddded	ed
ededef
dZddZdej                  de
e   fdZdefdZd Zdej                  defdZy)SileroSpeechGatezCStateful speech gate using Silero VAD probabilities and hysteresis.r   r&   r(   r   r+   sample_ratestart_thresholdstop_thresholdstart_framesend_silence_msr   r   r   r   r   c          	         || _         || _        t        |      | _        t        |      | _        t        dt        |            | _        | j                         }t        dt        t        |dz  |z  |z                    | _
        t        |      | _        | j                         | _        d| _        d| _        d| _        y )Nr        @@Fr   )modelr   rb   r   r   r   r_   r   _chunk_sizeroundend_silence_framesr   chunker_initial_statestate	in_speech_speech_run_silence_run)rV   r   r   r   r   r   r   r   s           rX   rw   zSileroSpeechGate.__init__   s     
&$_5#N33|#45%%'
"%s5.61[@:MNO#
 -Z8((*
rZ   ry   Nc                     t        | j                               | _        | j                         | _        d| _        d| _        d| _        y )NFr   )r   r   r   r   r   r   r   r   rv   s    rX   resetzSileroSpeechGate.reset   s>    ,T-=-=-?@((*
rZ   r   c           	      L   g }| j                   j                  |      D ]  }| j                  |      }|| j                  r| j                  n| j
                  k\  }d}d}|r| xj                  dz  c_        d| _        nd| _        | xj                  dz  c_        | j                  s)| j                  | j                  k\  rd| _        d}d| _        | j                  r)| j                  | j                  k\  rd| _        d}d| _        |j                  t        ||||              |S )NFr   r   T)rg   rh   ri   rj   )r   r   _feed_probabilityr   r   r   r   r   r   r   r   rf   )rV   r   	decisionschunkrg   rh   startedendeds           rX   processzSileroSpeechGate.process   s   	\\&&w/E007K#'+~~##4;O;OI GE  A% $%!#$ !!Q&!>>d&6&6$:K:K&K!%$%!~~$"3"3t7N7N"N!&#$ # +'#*$)	3 0B rZ   c                 r    t        t        | j                  dd       dd       }t        t        |dd            S )Nconfig
branch_16kr   i   )getattrr   r_   )rV   branchs     rX   r   zSileroSpeechGate._chunk_size  s1    Xt<lDQ76<566rZ   c                 |    t        | j                  d      r&| j                  j                  | j                        S y )Ninitial_stater   )hasattrr   r   r   rv   s    rX   r   zSileroSpeechGate._initial_state  s1    4::/::++8H8H+IIrZ   r   c                     | j                   j                  || j                  | j                        \  }| _        t	        |d      rt        |j                               S t        t        j                  |      j                  d      d         S )Nr   itemr   r   )
r   feedr   r   r   rb   r   r   r   r   )rV   r   rg   s      rX   r   z"SileroSpeechGate._feed_probability  su    "&**//4::4+;+; #2 #
TZ ;'))+,,RZZ,44R8;<<rZ   r   )r[   r\   r]   r^   r_   rb   rw   r   r   r   r   rf   r   r   r   r   rd   rZ   rX   r   r      s    M "!% #! 	
    2#rzz #d3F.G #J7S 7
=rzz =e =rZ   r   c                   N    e Zd Zddddedee   fdZdej                  de	fd	Z
y)
SmartTurnEndpointDetectorr   Nr   	thresholdr   r   c                .    || _         || _        || _        y r   )r   r   r   )rV   r   r   r   s       rX   rw   z"SmartTurnEndpointDetector.__init__   s     
&"rZ   audiory   c                     | j                   j                  || j                  | j                        }t	        t        |j                        t        |j                              S )Nr   )rm   rg   )	r   predict_endpointr   r   rl   rc   
predictionrb   rg   )rV   r   results      rX   predictz!SmartTurnEndpointDetector.predict+  sV    ,,t//4>> - 
  &++,%@R@R:S
 	
rZ   )r[   r\   r]   r_   r   rb   rw   r   r   rl   r   rd   rZ   rX   r   r     s?    
 "%)	# 		#
 E?	#
RZZ 
,< 
rZ   r   c            	           e Zd Zddddddededed	efd
Zedefd       ZddZde	j                  ddfdZdee   fdZddZddZdddedefdZy)VoxtralRealtimeTranscriberrO   r   r    Ftranscription_delay_msmax_decode_tokens_per_stepmax_turn_tokensrL   r   r   r   rL   c                    || _         || _        || _        || _        || _        d | _        d| _        d| _        d| _        d| _	        y N r           F)
r   r   r   r   rL   sessiontextlast_finish_stepslast_finish_secondslast_finish_hit_max_steps)rV   r   r   r   r   rL   s         rX   rw   z#VoxtralRealtimeTranscriber.__init__5  sP     
&<#*D'.	!"#& ).&rZ   ry   c                 R    | j                   d uxr t        | j                   dd       S )NdoneF)r   r   rv   s    rX   activez!VoxtralRealtimeTranscriber.activeI  s&    ||4'Tfe0T,TTrZ   Nc                     | j                   j                  | j                  | j                        | _        d| _        d| _        d| _        d| _        y )N)
max_tokensr   r   r   r   F)	r   create_streaming_sessionr   r   r   r   r   r   r   rv   s    rX   startz VoxtralRealtimeTranscriber.startM  sP    zz::++#'#>#> ; 
 	!"#& ).&rZ   r   c                    | j                   t        | j                   dd      r| j                          | j                   j                  t	        j
                  |t        j                        j                  d             y )Nr   Fr   r   )r   r   r  r   r   r   r   r   )rV   r   s     rX   r   zVoxtralRealtimeTranscriber.feedW  sP    <<74<<#GJJL"**WBJJ?GGKLrZ   c                     | j                   g S | j                   j                  | j                        }|D ]  }| xj                  |z  c_         |S )N)max_decode_tokens)r   stepr   r   rV   deltasdeltas      rX   r  zVoxtralRealtimeTranscriber.step\  sK    <<I""T5T5T"UEIII rZ   c                 R    | j                   | j                   j                          y y r   )r   closerv   s    rX   r  z VoxtralRealtimeTranscriber.closed  s!    <<#LL  $rZ   c                     | j                   | j                   j                          d | _         d| _        d| _        d| _        d| _        y r   )r   r  r   r   r   r   rv   s    rX   r   z VoxtralRealtimeTranscriber.reseth  sB    <<#LL 	!"#& ).&rZ   r"   	max_stepsr  c                   | j                   | j                  j                         S t        j                         }| j                          d}t        | j                   dd      s7||k  r2| j                          |dz  }t        | j                   dd      s||k  r2|| _        t        j                         |z
  | _	        ||k\  xr t        | j                   dd       | _
        | j                  rD| j                  r8t        j                  d|t        | j                  j                                      | j                  j                         }d | _         |S )Nr   r   Tr   z:event=voxtral_finalization_max_steps max_steps=%s chars=%s)r   r   striptime	monotonicr  r   r  r   r   r   rL   loggerwarninglen)rV   r  
start_timesteps
final_texts        rX   finishz!VoxtralRealtimeTranscriber.finishq  s   <<99??$$^^%


$,,5%):KIIKQJE $,,5%):K "'#'>>#3j#@ ).)); *
GLL&$E
 A
& <<D::NNLDIIOO%&
 YY__&
rZ   r   )r[   r\   r]   r_   rc   rw   propertyr   r  r   r   r   r   ra   r  r  r   r  rd   rZ   rX   r   r   4  s    
 '**+"/ !$	/
 %(/ / /( U U U/MBJJ M4 M
d3i !/ *, 3  rZ   r   c                   F    e Zd ZdedefdZd
dZddedeee      defd	Z	y)LocalLLMResponseEngine
model_namer7   c                <    || _         || _        d | _        d | _        y r   )r  r7   llm	tokenizer)rV   r  r7   s      rX   rw   zLocalLLMResponseEngine.__init__  s    $*rZ   ry   Nc                 H    t        | j                        \  | _        | _        y r   )load_llmr  r  r   rv   s    rX   r   zLocalLLMResponseEngine.load  s    #+DOO#< $.rZ   
transcriptcontextc                 `   | j                   | j                  | j                          d| j                  dg}|r|j	                  |       |j                  d|d       | j                  j                  |ddd      }t        | j                   | j                  |d      j                         S )NsystemrolecontentuserFT)tokenizeenable_thinkingadd_generation_prompt)rL   )	r  r   r   r7   extendr   apply_chat_templategenerate_textr  )rV   r#  r$  messagesprompts        rX   r	   zLocalLLMResponseEngine.generate  s    88t~~5IIK%$2D2DEFOOG$J?@33ueSW 4 
 TXXt~~vuMSSUUrZ   r   r   )
r[   r\   r]   ra   rw   r   r   r   dictr	   rd   rZ   rX   r  r    sB    3 # =
V3 
V$t*1E 
VQT 
VrZ   r  c                   F    e Zd Zdddddededee   fdZd	ed
ee   fdZ	y)PocketTTSResponderr:   rS   Nvoicestreaming_intervaltemperaturer7  r8  r9  c                z    || _         || _        || _        || _        t	        t        |dd      xs d      | _        y )Nr   i]  )r   r7  r8  r9  r_   r   r   )rV   r   r7  r8  r9  s        rX   rw   zPocketTTSResponder.__init__  s=     

"4&wumVDNOrZ   r   ry   c                     || j                   d| j                  d}| j                  | j                  |d<   t         | j                  j
                  di |      S )NT)r   r7  r   r8  r9  rd   )r7  r8  r9  iterr   r	   )rV   r   kwargss      rX   create_generatorz#PocketTTSResponder.create_generator  s]    ZZ"&"9"9	"
 '$($4$4F=!'DJJ''1&122rZ   )
r[   r\   r]   ra   rb   r   rw   r   r   r>  rd   rZ   rX   r5  r5    sO    
 $('+P 	P
 "P e_P	3S 	3Xc] 	3rZ   r5  c                       e Zd ZddddedefdZdeeef   fdZ	d	d
deeef   fdZ
edefd       Zdeeef   fdZddee   deeef   fdZdej$                  dedededededeeef   fdZddZy)AudioOutputStreamTN)enabledevent_callbackr   rA  c                X    || _         || _        |rt        ||      | _        y d | _        y )N)r   rB  )rA  r   r   player)rV   r   rA  rB  s       rX   rw   zAudioOutputStream.__init__  s5    &  KO 	  	rZ   ry   c           	      z   | j                  |      }| j                  !d| j                  ||| j                  z  dz  dS | j                  j                  }| j                  j	                  |       | j                  j                         }d| j                  j                  | xr | j                  j                  | j                  ||| j                  z  dz  ||| j                  z  dz  d}t        | j                  d      r9| j                  j                         |d<   | j                  j                  dz  |d<   |S )NFr   )player_enabledr   r   duration_msT)rF  player_playingplayer_startedr   r   rG  buffered_samplesbuffered_msstart_buffer_samplesstart_buffer_ms)	_sample_countrD  r   playingqueue_audiorJ  r   rL  start_buffer_seconds)rV   r   r   was_playingrJ  statuss         rX   rP  zAudioOutputStream.queue_audio  s&   $$U+;;"'#//"&)9)99FB	  kk))&;;779""kk11"-oE$++2E2E++"T%5%55> 0+d.>.>>G	
 4;; 67-1[[-M-M-OF)*(,(H(H6(QF$%rZ   Fforcec                v   | j                   ddd| j                  dd|dS t        | j                   d      r| j                   j                  |      S d| j                   j                  d| j                  | j                   j                         | j                   j                         | j                  z  dz  |dS )	NFr   r   )rF  rH  rI  r   rJ  rK  forcedstart_if_bufferedrT  Tr   )rD  r   r   rX  rO  rJ  )rV   rU  s     rX   start_playback_if_bufferedz,AudioOutputStream.start_playback_if_buffered  s    ;;"'"'"'#//$%"  4;; 34;;00u0==""kk11#++ $ < < >;;779D<L<LLvU
 	
rZ   c                 L   t        j                  |       }|j                  dk(  ry|j                  dk(  rt        |j                  d         S |j                  dk(  r|j                  d   dk(  rt        |j                  d         S |j                  d   dk(  rt        |j                  d         S |j                  d   dk  r7|j                  d   |j                  d   k  rt        |j                  d         S t        |j                  d         S t        |j
                        S )Nr   r   rG      )r   r   ndimr_   r   r   )r   r   s     rX   rN  zAudioOutputStream._sample_count  s    **U#<<1<<1w}}Q'((<<1}}Q1$7==+,,}}Q1$7==+,,}}Q1$q)9GMM!<L)L7==+,,w}}Q'((7<<  rZ   c                 \    | j                   | j                   j                         S dddd dS )NFr   r   )rR  rJ  rK  last_output_age_ms)rD  flushrv   s    rX   r_  zAudioOutputStream.flush	  s5    ;;";;$$&&  !"&	
 	
rZ   now_msc                 z    | j                   dd| j                  ddd d dS | j                   j                  |      S )NFr   r   )rF  rH  r   rJ  rK  !last_output_callback_monotonic_msr^  r`  )rD  r   playback_state)rV   r`  s     rX   rd  z AudioOutputStream.playback_state  sK    ;;"'"'#//$%"59&*  {{)))88rZ   r   r   input_end_msmin_delay_msmax_delay_msstep_msc                h    | j                   dd d dS | j                   j                  ||||||      S )Nr   correlationdelay_msmethodr   re  rf  rg  rh  )rD  echo_correlation)rV   r   r   re  rf  rg  rh  s          rX   ro  z"AudioOutputStream.echo_correlation   sI     ;;#&DDII{{++/%%% , 
 	
rZ   c                 R    | j                   | j                   j                          y y r   )rD  stoprv   s    rX   rq  zAudioOutputStream.stop5  s!    ;;"KK #rZ   r   r   )r[   r\   r]   r_   rc   rw   r3  ra   r   rP  rY  staticmethodrN  r_  r   rb   rd  r   r   ro  rq  rd   rZ   rX   r@  r@    s    <@QU 
s 
T 
DcN 6 38 
DcN 
. ! ! ! 
tCH~ 
9Xe_ 9S#X 9

 	

 
 
 
 
 
c3h
*rZ   r@  c                      e Zd ZdZ	 dCddddddddee   dee   dee   dee   dee	   f
d	Z
d
ededdfdZededefd       Zdee   deddfdZd
edeeef   ddfdZdCdee   deeef   fdZ	 dDdeeeef      dee   defdZdej2                  dedeeef   fdZededefd       Zdedeeef   fdZedededefd       Zd Zd  Z d! Z!dej2                  fd"Z"d# Z#dej2                  fd$Z$dedej2                  fd%Z%deddfd&Z&d'edefd(Z'	 dCdedeej2                     defd)Z(dddd*dedej2                  d+ed,ee   d-ee   d.eeeef      ddfd/Z)dd0d+ededej2                  d,ee   ddf
d1Z*defd2Z+deddfd3Z,defd4Z-d5 Z.d6 Z/d7 Z0d8 Z1d9 Z2d,efd:Z3ded;e4jj                  fd<Z6d
ed=e7d>e7ddfd?Z8d@eddfdAZ9dB Z:y)EVoicePipelinezLRealtime voice assistant pipeline using Voxtral Realtime STT and Pocket TTS.N)speech_gateendpoint_detectortranscriberresponse_enginetts_responderaudio_outputr   rw  rx  ry  rz  c                   |xs
 t               | _        || _        || _        || _        || _        || _        || _        t        j                  | j                  j                        | _        t        j                         | _        t        j                  | j                  j                        | _        t               | _        d | _        d | _        d | _        d| _        g | _        g | _        d | _        d | _        d | _        d | _        d| _        d| _        t9        | j                  j:                  | j                  j<                  z  dz        }t?        |      | _         y )N)maxsizer   Fr   )!r   r   ru  rv  rw  rx  ry  rz  r   QueuerK   input_audio_queuetranscript_queueoutput_audio_queuero   mlxr   _current_tts_task_current_tts_cancel_current_response_text_conversation_turn_audio_turn_started_at_candidate_silence_started_at_barge_candidate_started_at_barge_candidate_reason_barge_candidate_waiting_logged_tts_output_started_loggedr_   r   r/   r   _preroll)	rV   r   ru  rv  rw  rx  ry  rz  preroll_sampless	            rX   rw   zVoicePipeline.__init__=  s4    5 3 5&!2&.*(<CMMKK**=
 5<MMO=D]]KK**>
 $%9=	9=<@ &(#)+-/15>B*<@(6:$/4,*/'KK))DKK,B,BBVK
 &o6rZ   eventfieldsry   c                       j                   j                  sy |r@dj                   fd|j                         D              }t        j                  d||       y t        j                  d|       y )N c              3   P   K   | ]  \  }}| d j                  |         yw)=N)_format_log_value).0keyvaluerV   s      rX   	<genexpr>z+VoicePipeline._log_event.<locals>.<genexpr>n  s4      '"0JC %q//678"0s   #&zevent=%s %szevent=%s)r   rL   joinitemsr  info)rV   r  r  rendered_fieldss   `   rX   
_log_eventzVoicePipeline._log_eventj  sZ    {{""!hh '"(,,.' O KKuo>J&rZ   r  c                     t        | t              rt        |       j                         S t        | t              r| dS t        | t              r!| rt        d | D              rt        |       S | S t        |       S )Nz.3fc              3   <   K   | ]  }|j                           y wr   )isspace)r  chars     rX   r  z2VoicePipeline._format_log_value.<locals>.<genexpr>  s     #EutDLLNus   )
isinstancerc   ra   rT   rb   anyrepr)r  s    rX   r  zVoicePipeline._format_log_valuev  ss    eT"u:##%%eU#C[!eS! #Eu#E E U 
 5zrZ   r   nowc                    | j                   j                  r|sy t        d |D              }|sy |d   }t        d |D              }| j	                  d|dz  |j
                  |t        |      t        d |D              t        t        | j                  dd            t        d	 |D              t        d
 |D              	       y )Nc              3   P   K   | ]  }|j                   xs |j                     y wr   )ri   rj   r  decisions     rX   r  z3VoicePipeline._log_vad_decisions.<locals>.<genexpr>  s,      
% ##?x'?'??%s   $&r   c              3   4   K   | ]  }|j                     y wr   )rg   r  s     rX   r  z3VoicePipeline._log_vad_decisions.<locals>.<genexpr>  s     M9xh229   vadr   c              3   4   K   | ]  }|j                     y wr   )rh   r  s     rX   r  z3VoicePipeline._log_vad_decisions.<locals>.<genexpr>  s     KXh00r  r   Fc              3   4   K   | ]  }|j                     y wr   )ri   r  s     rX   r  z3VoicePipeline._log_vad_decisions.<locals>.<genexpr>  s     Qy8x66yr  c              3   4   K   | ]  }|j                     y wr   )rj   r  s     rX   r  z3VoicePipeline._log_vad_decisions.<locals>.<genexpr>  s     SX 8 8r  )monotonic_msrg   max_probabilityframesspeech_framesr   ri   rj   )r   rL   r  r   r  rg   r  sumrc   r   ru  )rV   r   r  has_transitionlastr  s         rX   _log_vad_decisionsz VoicePipeline._log_vad_decisions  s     {{"") 
%
 
 }M9MMv((+y>KKK74#3#3[%HIQyQQSSS 	 
	
rZ   c                      j                   j                  sy dk(  r' j                  r|j                  d      sy d _        dt	        |      j                  dt        j                         dz          j                  ; j                  j                         r! j                  j                   fd       y   j                  fi  y )Ntts_output_callbackrS  Ttts_output_startedr  r   c                  *     j                    fi S r   )r  )r  
log_fieldsrV   s   rX   <lambda>z6VoicePipeline._handle_playback_event.<locals>.<lambda>  s    ?4??53WJ3WrZ   )r   rL   r  rU   r3  
setdefaultr  r  r   
is_runningcall_soon_threadsafer  )rV   r  r  r  s   `` @rX   _handle_playback_eventz$VoicePipeline._handle_playback_event  s    {{""))..vzz(7K.2D+(E&\
ndnn.>.GH99 TYY%9%9%;II**+WXDOOE0Z0rZ   c                     ||nt        j                         dz  }| j                  t        | j                  d      sddddd dS | j                  j	                  |      S )Nr   rd  Fr   r   )rF  rH  rJ  rK  r^  rc  )r  r  rz  r   rd  )rV   r  r`  s      rX   _playback_statezVoicePipeline._playback_state  so    #dnn.>&H$G/-
 #("'$%"&*    //v/>>rZ   r   c                     |xs | j                  |      }|j                  d      }|d uxr- dt        |      cxk  xr | j                  j                  k  nc }t        |j                  d      xs |      S )Nr^  g      IrH  )r  rU   rb   r   rB   rc   )rV   r   r  r^  recent_outputs        rX   _playback_echo_window_activez*VoicePipeline._playback_echo_window_active  sz     2--c2"YY';<d* 3'(3{{223 	 EII./@=AArZ   r   c                 :   | j                   t        | j                   d      sdd d dS | j                   j                  || j                  j                  |dz  | j                  j
                  | j                  j                  | j                  j                        S )Nro  r   rj  r   rn  )rz  r   ro  r   r   rC   rE   rF   )rV   r   r  s      rX   _echo_correlationzVoicePipeline._echo_correlation  s    $G1-
 $'DDII  11"kk;;v6666KK88 2 
 	
rZ   r   c                 h    dj                  t        j                  d| j                                     S )Nr  z	[a-z0-9]+)r  refindallrT   r   s    rX   _normalize_text_for_matchz'VoicePipeline._normalize_text_for_match  s"    xx

<>??rZ   c                     | j                  |      }t        |j                  dd            }|| j                  j                  k  ry| j                  | j
                        }|r| j                  ||      ryy)Nr  r   )	too_shortbelow_min_chars)echotranscript_matches_playback)speechr#  )r  r  replacer   rH   r  _looks_like_playback_text)rV   r   	candidatecompact_lenresponses        rX   _barge_candidate_text_statusz*VoicePipeline._barge_candidate_text_status  sl    2248	)++C45BBB111$2M2MN66y(K8%rZ   r  r  c                 \   | r|sy|j                  |       s| |v ry| j                         }|j                         }|rt        |      t        |      kD  ryt        t        |      t        |      z
  dz         D ]1  }|||t        |      z    }t	        d t        ||      D              s1 y y)NFTr   c              3   D   K   | ]  \  }}|j                  |        y wr   )
startswith)r  candidate_tokenresponse_tokens      rX   r  z:VoicePipeline._looks_like_playback_text.<locals>.<genexpr>  s(      7T3O^ ))/:7Ts    )r  splitr  rangeallzip)r  r  candidate_tokensresponse_tokensindexwindows         rX   r  z'VoicePipeline._looks_like_playback_text  s    y)Y(-B$??,"..*3'7#83;O#O3/#6F2GG!KLE$UUS9I5J-JKF 7:;KV7T   M rZ   c                 D   K    j                   3ddlm 	  j                  j	                   fd       d {    _          j                  3ddlm 	  j                  j	                   fd       d {    _	         j                  ddlm  j                  j	                   fd       d {   }t        |t         j                  j                         j                  j                    j                  j"                   j                  j$                         _         j&                  qt)         j                  j*                   j                  j,                  	       _         j                  j	                   j&                  j                         d {     j.                   j                  j	                   fd
       d {   }t1        | j                  j2                  t5         j                  j6                         j                  j8                         _         j:                  i j                  j<                  xs  j.                  j>                  }tA        t        |       j                  jB                   jD                         _        y y 7 n# t
        $ r(}t        d j                  j                        |d }~ww xY w7 f# t
        $ r(}t        d j                  j                        |d }~ww xY w7 _7 w7 Hw)Nr   r
   c                  "   t          j                  j                        j                  j                  j                  j                  j                  j
                  j                  j                  j                  j                        S )Nr   )r   r   r%   r   r'   r)   r*   r,   load_vadrV   s   rX   r  z+VoicePipeline.init_models.<locals>.<lambda>  s^    , !6!67$(KK$A$A(,(G(G'+{{'E'E%)[[%A%A'+{{'E'ErZ   z Failed to load speech VAD model c                      t          j                  j                        j                  j                  j                  j                        S )Nr   )r   r   r1   r   r3   r  s   rX   r  z+VoicePipeline.init_models.<locals>.<lambda>  s7    5 !7!78$(KK$A$A"&++"<"<rZ   z Failed to load Smart Turn model c                  <      j                   j                        S r   )r   r   )load_sttrV   s   rX   r  z+VoicePipeline.init_models.<locals>.<lambda>  s    Xdkk6K6K-LrZ   r   )r7   c                  B    t         j                  j                        S r   )load_ttsr   r9   rv   s   rX   r  z+VoicePipeline.init_models.<locals>.<lambda>/  s    Xdkk6K6K-LrZ   r6  )r   rA  rB  )#ru  mlx_audio.vadr   r  r   	ExceptionRuntimeErrorr   r%   rv  r1   rw  mlx_audio.stt.utilsr   r_   r   r   r!   rL   rx  r  r6   r7   ry  r5  r;   rb   r<   r=   rz  r   r   r@  rI   r  )rV   excstttts	output_srr  r  s   `    @@rX   init_modelszVoicePipeline.init_models  s~    #6)-	* 	$  !!)6/3xx||0 *& #<%LMMC9'*4;;+Q+Q'R+/;;+U+U $ ? ?++ D '#9**$++:S:S$D  ((,,t3388999%%LMMC!3kk++#()K)K#L KK77	"D $66X$:L:L:X:XI 1	N..#::!D %u	$  "6t{{7L7L6OP*  "6t{{7M7M6PQ N : Ns   L "J, J)	J, L "K# :K ;	K# 4L 8L9C*L #L$1L LCL )J, ,	K5#KKL  K# #	L,#LLL L L c                   K   t        j                         | _        g }	 | j                          d {    t        j                  | j                               t        j                  | j                               t        j                  | j                               t        j                  | j                               g}t        j                  |  d {    |D ]  }|j                           t        j                  |ddi d {    | j                  | j                  j                          | j                  j                          d {    y 7 &7 7 R7 # |D ]  }|j                           t        j                  |ddi d {  7   | j                  | j                  j                          | j                  j                          d {  7   w xY ww)Nreturn_exceptionsT)r   r   r   r  create_task	_listener_transcription_stepper_response_processor_audio_output_processorgathercancelrz  rq  r  r   )rV   taskstasks      rX   r  zVoicePipeline.start?  s~    ,,.		&""$$$##DNN$45##D$?$?$AB##D$<$<$>?##D$@$@$BC	E ..%((( ..%@4@@@  ,!!&&(((##%%% % ) A &  ..%@4@@@  ,!!&&(((##%%%s}   G$E! EB#E! EE! /G$
EAG$EG$E! E! G$G$!0G!FAG!GG!!G$c                      |r j                  dt        |              j                  |       fd} j                   j                  j	                  |       y y )Ninput_status)rS  c                      	 j                   j                          y # t        j                  $ r0 j                  j
                  rt        j                  d       Y y Y y w xY w)Nz*Dropping input audio because queue is full)r~  
put_nowaitr   	QueueFullr   rL   r  debugr   rV   s   rX   _enqueuez,VoicePipeline._sd_callback.<locals>._enqueueX  sS    O&&11%8$$ O;;&&LL!MN 'Os    >A"!A")r  ra   _normalize_inputr   r  )rV   indatar  _timerS  r  r   s   `     @rX   _sd_callbackzVoicePipeline._sd_callbackS  sS    OON3v;O?%%f-	O 99 II**84 !rZ   c                 ,   t        j                  |      }t        j                  |j                  t         j                        r^t        t        t        j                  |j                        j                              }|j                  t         j                        |z  }nt        j                  |j                  t         j                        r`t        j                  |j                        }t        |j                        dz   dz  }|j                  t         j                        |z
  |z  }n|j                  t         j                        }|j                  dkD  r|j                  d      }t        j                  |j!                  d      dd      S )Ng      ?g       @r   )axisr   g      )r   r   
issubdtyper   signedintegerrb   absiinfominr   r   unsignedintegerr   r\  meanclipr   )rV   r  r   scaler  midpoints         rX   r  zVoicePipeline._normalize_inputb  s   

6"==b&6&67#bhhu{{37789ELL,u4E]]5;;(:(:;88EKK(Ddhh#-4H\\"**-8HDELL,E::>JJAJ&Ewwu}}R($44rZ   c                 >  K   t        | j                  j                  | j                  j                  dz  z        }t	        j
                  | j                  j                  || j                  j                  d| j                        }|j                          | j                  d| j                  j                  | j                  j                  | j                  j                  | j                  j                         	 	 | j                  j                          d {   }| j                  |       d {    | j                  j                          V7 87 !# t         j"                  t$        f$ r  w xY w# |j'                          |j)                          w xY ww)Nr   int16)
samplerate	blocksizechannelsr   callback	listening)r   frame_msr'   r)   )r_   r   r   r   sdInputStreamr   r  r  r  r'   r)   r~  rU   _process_input_audio	task_doner   CancelledErrorKeyboardInterruptrq  r  )rV   
frame_sizer   r   s       rX   r  zVoicePipeline._listenerr  s@    KK))T[[-J-JV-ST

 {{44 [[//&&
 	55[[22 $ ? ?#{{== 	 	
		 $ 6 6 : : <<//888&&002 <8&&(9: 		 KKMLLNsB   C<F?E EE 6E7 E E E55E8 8"FFc                    K    j                   j                   fd       d {   }|s j                  j                         y t	        j
                         } j                  ||        j                  [ j                  j                  rE j                  j                          j                  j                  j                                n j                  j                         |D ]  }|j                  r j                  |       d {    n& j                   j                  ||       d {    |j                   s] j                   j#                  d|       d {     j                   j%                  |       d {      j                   j                  j                  rm j&                  `| j&                  z
   j(                  j*                  k\  r9 j-                  d| j&                  z
          j/                          d {    y y y y y 7 7 7 7 7 7 w)Nc                  :    j                   j                         S r   )ru  r   )r   rV   s   rX   r  z4VoicePipeline._process_input_audio.<locals>.<lambda>  s    t/?/?/G/G/PrZ   ended_before_confirmmax_turn_duration)duration_seconds)r  r   r  r   r  r  r  rw  r   r   r  r   ri   _handle_speech_startedr  _maybe_confirm_barge_candidaterj   _suppress_barge_candidate_maybe_finalize_turnr  r   r-   r  _finalize_turn)rV   r   r   r  r  s   ``   rX   r'  z"VoicePipeline._process_input_audio  s     ((,,'PQQ	MM  )nn	3/'D,<,<,C,C!!'*##GLLN3MM  )!H&&11#w???11=99(CHHH''33?88.W   %%133C888 " (  ''%%1d+++t{{/O/OOOO#!$t'<'<!<   %%''' P 2 ( )7 R @H 9 (sw   #I H2C0I H5&I ?H8 I "I 3H:4I I H<BI (H>)
I 5I 8I :I <I >I c                   K   | j                   0| j                   j                  r| j                  |       d {    y | j                  |      }| j	                  ||      r| j                  |       d {    | j                  ||      }| j                  d| j                  | j                  j                  t        |j                  d            t        |j                  d      xs d      |j                  d      |j                  dd      |j                  d      |j                  d      		       y | j                  |       d {    y 7 7 7 
w)
Nbarge_in_candidaterH  rK  r   r^  rk  rl  rm  )reasonr@   rH  rK  r^  ro  echo_delay_msecho_method)rw  r   _start_turnr  r  _start_barge_candidater  r  r  r   r@   rc   rU   rb   )rV   r  r   rd  rk  s        rX   r1  z$VoicePipeline._handle_speech_started  s2    'D,<,<,C,C""3'''--c2,,^SA--c22200#>KOO$33 $ ; ;#N$6$67G$HI!."4"4]"C"JsK#1#5#56J#K!,!D)ooj9'OOH5  
 s###% ( 3 	$s4   7EE<E6E7CEEEEEc                    K   || _         d| _        d| _        | j                  1| j                  j                  s| j                  |       d {    y y y 7 w)Nplayback_echo_windowF)r  r  r  rw  r   _start_transcriber_capturerV   r  s     rX   r<  z$VoicePipeline._start_barge_candidate  sX     +.('=$/4,'0@0@0G0G11#666 1H'6s   AAAAr  c                   K   | j                   |j                  sy || j                   z
  dz  }|| j                  j                  k  ry | j                  s0d| _        | j                  d|| j                  j                         y y w)Nr   Tbarge_in_waiting_for_transcript)
elapsed_ms	min_chars)r  rh   r   r@   r  r  rH   )rV   r  r  rC  s       rX   r2  z,VoicePipeline._maybe_confirm_barge_candidate  s      ++38;M;MD<<<F
3333337D0OO1%++CC   4s   BBc                   K   | j                   | j                  yt        | j                  dd      j                         }| j	                  |      \  }}|dk(  ry|}|@| j
                  r| j
                  d   n$t        j                  dt        j                        }|dk(  r| j                  ||||	       d {    y
| j                  ||||       d {    y
7 #7 w)NFr   r   r  r   r   r   r  r#  T)r8  r#  )r  rw  r   r  r  r  r   r   r   r3  _confirm_barge_candidate)rV   r  r   r   rS  r8  sample_windows          rX   $_evaluate_barge_candidate_transcriptz2VoicePipeline._evaluate_barge_candidate_transcript  s	     ++3t7G7G7Ot''4::<::4@[   ##   $XXarzz2 
 V00]t 1    ++	 , 
 	
 	
 
	
s$   B:C$<C =C$C"C$"C$)r#  rC  rk  r8  r#  rC  rk  c                \  K   | j                   y | j                   }|||z
  dz  }|| j                  ||      }|||j                  dd      |j                  d      |j                  d      d}|||d<   t        |      |d<    | j                  di | d | _         d | _        d	| _        | j                  j                  r| j                          d {    | j                  't        | j                  d
      rd| j                  _        | j                  /| j                  j                  s| j                  |       d {    || _        d | _        | j	                  d| j                  j$                  | j                  j&                  | j                  j(                         y 7 7 fw)Nr   rk  r   rl  rm  r8  rC  ro  r9  r:  r   charsFr   Tri   r/   r'   r*   )barge_in_confirmed)r  r  rU   r  r  r  r  r   r>   _handle_barge_inru  r   r   rw  r   r?  r  r  r/   r'   r*   )	rV   r  r   r8  r#  rC  rk  candidate_started_atr  s	            rX   rG  z&VoicePipeline._confirm_barge_candidate  s     ++3#?? 44>J00#>K$ +s C(__Z8&??84

 !!+Jv"%j/Jw;
;+/('+$/4,;;'')))'GD4D4Dk,R)-D&'0@0@0G0G112FGGG 4-1*{{-- $ ? ?![[99	 	 	
 * Hs&   CF,F(A-F,F*A%F,*F,rF  c                F   K    j                   y | j                   z
  dz  } j                  ||      }d  _         d  _        d _        g  _        d  _        d  _         j                  j                           j                  <t         j                  d      r& j                  j                   fd       d {    |||j                  dd      |j                  d      |j                  d      d	}|||d
<   t        |      |d<     j                  di | y 7 bw)Nr   Fr   c                  8     j                   j                         S r   )rw  r   rv   s   rX   r  z9VoicePipeline._suppress_barge_candidate.<locals>.<lambda>F      t'7'7'='='?rZ   rk  r   rl  rm  rK  r   rL  )barge_in_suppressed_echo)r  r  r  r  r  r  r  r  r   rw  r   r  r   rU   r  r  )rV   r8  r  r   r#  rC  rk  r  s   `       rX   r3  z'VoicePipeline._suppress_barge_candidate2  s     ++3D<<<F
,,Wc:+/('+$/4, $-1*'GD4D4Dg,N((,,?@@@$ +s C(__Z8&??84

 !!+Jv"%j/JwAjA As   B9D!<D=A#D!c                 P  K   d | _         d | _        d| _        | j                  | j                  j                  rd | _        y | j                  j                  r| j                          d {    | j                  't        | j                  d      rd| j                  _        | j                  y | j                  |       d {    || _        | j                  d| j                  j                  | j                  j                   | j                  j"                         y 7 7 _w)NFr   Tri   rM  )r  r  r  rw  r   r  r   r>   rO  ru  r   r   r?  r  r  r/   r'   r*   r@  s     rX   r;  zVoicePipeline._start_turnS  s     +/('+$/4,'D,<,<,C,C15D.;;'')))'GD4D4Dk,R)-D&#--c222 #{{-- $ ? ?![[99	 	 	
 *
 	3s&   A)D&+D",AD&D$AD&$D&c                    K    j                   y  j                   j                  s j                  j                   fd       d {     j                  j                         }g  _        |j                  rD j                   j                  |        j                  j                  |j                                d  _        y y 7 ~w)Nc                  8     j                   j                         S r   )rw  r  rv   s   rX   r  z:VoicePipeline._start_transcriber_capture.<locals>.<lambda>m  rS  rZ   )rw  r   r  r   r  rU   r  r   r   r   r   r  )rV   r  prerolls   `  rX   r?  z(VoicePipeline._start_transcriber_capturei  s     #&&((,,?@@@mm'')G!D||  %%g.  ''715D. '@s   AC	C	A?C	c                 ^   K    j                   sy  j                  | _        t        j                   j                         j	                  t        j
                  d       j                  j                   fd       d {   }| j                  z
  dz  } j                  d|j                  |j                  |j                   j                  j                  z  dz         |j                  s| j                  j                  k\  r j                          d {    y y 7 7 w)NFr   c                  :    j                   j                         S r   )rv  r   r  s   rX   r  z4VoicePipeline._maybe_finalize_turn.<locals>.<lambda>|  s    d.D.D.L.LU.SrZ   r   endpoint_candidate)rm   rg   
silence_msaudio_ms)r  r  r   r   r   r   r  r   r  rm   rg   r   r   r   r4   r5  )rV   r  r  r\  r   s   `   @rX   r4  z"VoicePipeline._maybe_finalize_turnu  s     --514D.t//077

7O&STTD>>>&H
 && ,,!ZZ$++"?"??&H 	 	
 T[[GGG%%''' H U (s%   BD-D)BD-"D+#D-+D-c                 |   K    j                    j                   j                   j                  dd       y  j                  r>t	        j
                   j                        j                  t        j                  d      n$t	        j                  dt        j                        }|j                   j                  j                  z  dz  }t        t         j                   d	d
            } j                  d|| j                  j                   j                  j                         t!        j"                         } j$                  j'                   fd       d {   }t!        j"                         |z
  dz  }t         j                   dd      }t         j                   dd      }g  _        d  _        d  _         j,                  j/                           j                  d|t        |      |||       |r$ j0                  j3                  |       d {    y  j                  dd       y 7 7 w)Nturn_finalization_skippedno_active_session)r8  Fr   r   r   r   r   r   turn_finalization_started)r]  partial_charsr  r   c                  d     j                   j                   j                  j                        S )Nr  )rw  r  r   r#   rv   s   rX   r  z.VoicePipeline._finalize_turn.<locals>.<lambda>  s(    D$$++++@@ , rZ   r   r   turn_finalized)r   rL  rC  finish_stepshit_max_stepsturn_droppedempty_transcript)rw  r   r  r  r   r   r   r   r   r   r   r   r  r   r#   r!   r  r  r  r   r  r  r  r   r  put)	rV   
turn_audioturn_audio_msrb  r  r   rC  re  rf  s	   `        rX   r5  zVoicePipeline._finalize_turn  s    #t'7'7'?'?'GOO7@SOT  NN4++,33BJJU3K!2::. 	
 #$++*G*GG&PGD$4$4fbAB'"'kk<< KK;; 	 	
 ^^%
XX\\
 

 nn&3v=
t//1DaH 0 02MuU $-1*d)!%' 	 	
 ''++D111OON3EOF/
* 2s%   EH<"H8#B;H<H:H<:H<c                 2  K   | j                   d uxr | j                   j                          }| j                  j                         }| j	                         }| j
                  | j
                  j                          |r| j                   j                          | j                          i }| j                  | j                  j                         xs i }t        |j                  d      xs d      }t        |j                  d      xs$ |j                  d      xs |j                  d            }|s|s|r\| j                  d|||t        |j                  d      xs d      t        |j                  d            |j                  d      	       y y w)
NrJ  r   rR  rH  r>   rK  r   r^  )cancelled_ttsflushed_chunksflushed_buffered_samplesflushed_buffered_msplayback_was_playingr^  )r  r   r  qsizer  r  setr  _clear_output_audio_queuerz  r_  r_   rU   rc   r  rb   )rV   
tts_activequeued_audiord  flush_statusro  playback_was_actives          rX   rO  zVoicePipeline._handle_barge_in  s    ""$.Tt7M7M7R7R7T3T 	 ..446--/##/$$((*""))+&&((,,224:L#&|'7'78J'K'Pq#Q "]+ 6!!"236!!"45

 )<OO(+)A$),*:*:=*I*PS$Q%),*:*:=*I%J#/#3#34H#I   *=s   FFc                     	 	 | j                   j                          | j                   j                          6# t        j                  $ r Y y w xY wr   )r  
get_nowaitr(  r   
QueueEmptyrv   s    rX   rt  z'VoicePipeline._clear_output_audio_queue  sO    4''224 ''113  %% s   8 AAc                    K   	 t        j                  d       d {     j                   j                  j                  sA j                  j                   fd       d {   }|D ]  }|s j                  d|         j                  + j                  t        j                                d {    7 7 [7 	w)Ng{Gz?c                  8     j                   j                         S r   )rw  r  rv   s   rX   r  z6VoicePipeline._transcription_stepper.<locals>.<lambda>  s    0@0@0E0E0GrZ   partial_transcript_deltar  )r   sleeprw  r   r  r   r  r  rI  r  r  r  s   `  rX   r  z$VoicePipeline._transcription_stepper  s     --%%%'t/?/?/F/F88<<(GHHFOO$>UOK   //;??@PQQQ % I
 Rs;   CC AC&C'C3AC:C;CCCc                 *  K   	 | j                   j                          d {   }t        j                  | j	                  |            }	 | d {    | j                   j                          l7 N7 !# | j                   j                          w xY wwr   )r  rU   r   r  _respond_to_transcriptr(  )rV   r#  r  s      rX   r  z!VoicePipeline._response_processor  sz     #4488::J&&t'B'B:'NOD2

%%//1 : %%//1s9   BA0(BA4 A2A4 B2A4 4BBc                    K    j                   j                   fd       d {   }|j                         }|sy | _         j	                  d|t        |              j                  j                  dd        j                  j                  d|d       t        j                          _
        t        j                   j                  | j                               _        y 7 ŭw)Nc                  P     j                   j                   j                        S r   )rx  r	   r  )rV   r#  s   rX   r  z6VoicePipeline._respond_to_transcript.<locals>.<lambda>  s    D((11*d>P>PQrZ   response_ready)r   rL  r*  r'  	assistant)r  r   r  r  r  r  r  r   r   Eventr  r  _speak_responser  )rV   r#  response_texts   `` rX   r  z$VoicePipeline._respond_to_transcript  s     "hhllQ
 
 &++-&3#(}CDVW!!6j"IJ!!;="QR#*==? !(!4!4  0H0HI"

s   #C/C-CC/cancel_eventc                   	
K   | j                  dt        |             d| _        | j                  j	                  |      	t               
d}d}d}	
fd}	 |j                         s| j                  j                  |       d {   }|
u rd}ne|j                         rnT| j                  j                  |       d {    |dz  }|t        |j                  d         z  }|j                         s|rD|j                         s4| j                  j                          d {    | j                  d       | j                  |rd	nd
||       y 7 7 7 2# t         j"                  $ r& |j%                          | j                  d
||       Y y t&        $ r&}| j                  dt)        |             Y d }~y d }~ww xY ww)Ntts_started)rL  Fr   c                  |    	 t              } t        j                  | j                        S # t        $ r cY S w xY wr   )nextStopIterationr   r   r   )r   	generatorsentinels    rX   _next_chunkz2VoicePipeline._speak_response.<locals>._next_chunk  s<     i ::fll++ !   s   - ;;Tr   rT  tts_finishedtts_cancelled	tts_error)error)r  r  r  ry  r>  objectis_setr  r   r  ri  r_   r   r  _start_buffered_tts_playback_log_tts_doner   r)  rs  r  ra   )rV   r   r  finishedchunk_countsample_countr  r   r  r  r  s            @@rX   r  zVoicePipeline._speak_response  s    SY7*/'&&77=	8	,	"))+#xx||K88X%#H&&(--11&999q FLLO 44 #))+  3 3 5--22444111= "*% 9 : 5%% 	K\J 	9OOKs3xO88	9sx   AG.E E	:E E4E 9/E (E)E ?GE E E 6GGGF>9G>GGr  r  c                     t        t        | j                  dd      xs d      }||d}|dkD  r||z  dz  |d<    | j                  |fi | y )Nr   r   )r   r   r   rG  )r_   r   ry  r  )rV   r  r  r  r   r  s         rX   r  zVoicePipeline._log_tts_done.  s]    '$"4"4mQGL1M!#"
 ?$0;$>$GF=!((rZ   rU  c                   | j                   t        | j                   d      sy | j                   j                  |      xs i }| j                  j                  r|j                  d      sy | j                  d|j                  d      |j                  d      |j                  d      |j                  d      t        |j                  d	            | j                  j                         t        j                         d
z         y )NrY  rT  rI  tts_playback_startedr   rJ  rK  rM  rW  r   r   rJ  rK  rM  rW  output_queue_sizer  )rz  r   rY  r   rL   rU   r  rc   r  rr  r  r  )rV   rU  rS  s      rX   r  z*VoicePipeline._start_buffered_tts_playback8  s    $G;-
 ""==E=JPb{{""&**5E*F"

=1#ZZ(:;

=1"JJ'89

8,-"55;;=)F2 	 		
rZ   c                   K   	 | j                   j                          d {   }| j                  t        j                         }| j                  j                  |      xs i }| j                  j                  r|j                  d| j                   j                                |j                  d|dz         |j                  d      rs| j                  d|j                  d      |j                  d      |j                  d      |j                  d	      d
|j                  d      |j                  d             | j                   j                          W7 :w)Nr  r  r   rI  r  r   rJ  rK  rM  Fr  )r  rU   rz  r  r  rP  r   rL   r  rr  r  r(  )rV   r   	queued_atrS  s       rX   r   z%VoicePipeline._audio_output_processorK  s$    115577E  , NN,	**66u=C;;&&%%+T-D-D-J-J-L %%ni&6HIzz"232(.

=(A-3ZZ8J-K(.

=(A,2JJ7H,I#(.4jj9L.M)/N)C ( 	 ##--/+ 7s   EED:Er   )NN);r[   r\   r]   r^   r   r   r   r   r5  r@  rw   ra   r  rr  r  r   rf   rb   r  r3  r  r  rc   r  r   r   r  r  tupler  r  r  r  r  r  r  r'  r1  r<  r2  rI  rG  r3  r;  r?  r4  r5  rO  rt  r  r  r  r   r  r  r_   r  r  r   rd   rZ   rX   rt  rt  :  s   V 15+7 <@)-6:48+7,-+7 89+7 "#+7   23+7 01+7Z
' 
's 
't 
'    
12
9>
	
21C 1c3h 1D 1?8E? ?d38n ? NRBd38n-B;CE?B	B
 
% 
DcN 
 @ @ @ @	& 	&sCx 	& S C D  &EN&(55"** 5 <%("** %(N$ $

 $,7 7$ 7+27" ;?#+BJJ#7	J %)&*04-
-
 -

 -
 SM-
 UO-
 d38n--
 
-
j %)BB B 	B SMB 
BB
U 
,
6E 
6d 
6(e (,*GX>4
R2
s 
 (# (W]] (T)3 )S ) )PT )
T 
d 
&0rZ   rt  c                    K   t        j                  dt         j                        } | j                  dt        dd       | j                  dt        dd	       | j                  d
t        dd       | j                  dt        dd       | j                  dt        dd       | j                  dg ddd       | j                  dt        dd       | j                  dt
        d d       | j                  dt
        dd       | j                  d t
        d!d"       | j                  d#t        d d$       | j                  d%t        d&d'       | j                  d(t        d)d*       | j                  d+t
        d,d-       | j                  d.t
        d/d0       | j                  d1t        d2d3       | j                  d4t
        d5d6       | j                  d7t
        d8d9       | j                  d:t
        d;d<       | j                  d=t
        d>d?       | j                  d@t
        dAdB       | j                  dCdDdEdFG       | j                  dHdDdIJ       | j                         }|j                  r1t        j                         j                  t        j                         t        dbi dK|j                  dL|j                  dM|j                   dN|j"                  dO|j$                  dP|j&                  dQ|j(                  dR|j*                  dS|j,                  dT|j.                  dU|j0                  dV|j2                  dW|j4                  dX|j6                  dY|j8                  dZ|j:                  d[|j<                  d\|j>                  d]|j@                  d^|jB                  d_|jD                  d`|j                  da|jF                   }tI        |      }|jK                          d {    y 7 w)cNzVoice Pipeline)descriptionformatter_classz--stt_modelr   z	STT model)typedefaulthelpz--tts_modelr8   z	TTS modelz--llm_modelr5   z	LLM modelz--vad_modelr$   zSpeech trigger VAD modelz--turn_modelr0   zSmart Turn endpoint modelz--latency_profilerP   r   zLatency profile)choicesr  r  z--voicer:   zPocket TTS voicez--transcription_delay_msz-Override Voxtral Realtime transcription delayz--stt_max_turn_tokensr    z+Maximum Voxtral decode tokens per user turnz--stt_finalization_max_stepsr"   z<Maximum bounded Voxtral finalization steps after endpointingz--tts_streaming_intervalz.Pocket TTS streaming chunk interval in secondsz--vad_start_thresholdr&   z-Silero VAD probability needed to start a turnz--vad_stop_thresholdr(   z9Silero VAD probability needed to keep an active turn openz--vad_start_framesr   z7Consecutive speech frames needed before starting a turnz--vad_end_silence_msr+   z5Silence duration before asking Smart Turn to finalizez--turn_thresholdr2   z)Smart Turn endpoint probability thresholdz--min_barge_in_msr?   zASpeech duration required before confirming playback-time barge-inz--ignore_playback_echo_msrA   z2Playback echo window after recent output callbacksz--echo_delay_min_msr.   zBMinimum expected acoustic echo delay for output-reference matchingz--echo_delay_max_msrD   zBMaximum expected acoustic echo delay for output-reference matchingz--barge_in_min_transcript_charsrG   z@Minimum partial transcript characters needed to confirm barge-inz	--verbose
store_trueFz+Enable structured voice pipeline debug logs)actionr  r  z	--no_playz#Run without playing generated audio)r  r  r   r   r9   r;   r6   r%   r1   r'   r)   r*   r,   r3   r@   rB   rC   rE   rH   r   r!   r#   r<   rL   rI   rd   )&argparseArgumentParserArgumentDefaultsHelpFormatteradd_argumentra   r_   rb   
parse_argsrL   logging	getLoggersetLevelINFOr   r   r   r9   r7  	llm_modelr%   r1   r'   r)   r*   r,   r3   r@   rB   rC   rE   rH   r   r!   r#   r<   no_playrt  r  )parserargsr   pipelines       rX   mainr  d  s    $$$ >>F B	   *	   C	   *'	   -(	   /	   	   "<	   :	   &K	   "=	   <	   H	   F	   D	   8	   P	   #A	   Q	   Q	   )O	   :	   2  
 D||$$W\\2  ,,.. .. **	
 ~~ .. ?? !44  22 ..  22 ** ,, !% < < 00  00!" '+&H&H#$ $(#>#>%& !44'( $(#B#B)*  $::+, -. ||#/F2 V$H
..
s   O:P<P=P__main__)5r  r   r  r  r  concurrent.futuresr   dataclassesr   typingr   r   r   r   mlx.corecorer{   numpyr   sounddevicer%  mlx_lm.generater	   r0  mlx_lm.utilsr   r"  mlx_audio.sts.audio_playerr   mlx_audio.tts.utilsr   r  basicConfigWARNINGr  r[   r  r   rf   rl   ro   r   r   r   r   r   r  r5  r@  rt  r  r   r*  rd   rZ   rX   <module>r     sd      	  1 ! 4 4    5 ) 2 6   
//"X 
		8	$ ?! ?! ?!D " " "   
#A #AL $   FW= W=t
 
*S SlV V.3 36{ {|g0 g0Tl^ zDF   s   D6 6D>=D>