
    GJjaE                         d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d	d
lmZmZmZmZ  G d d      Z  G d de       Z!d Z"d Z#d Z$d Z%d Z&d Z'd Z(y)    N)Counter)chain)Path)Empty)Queue)select)PIPEPopenrun   )Hostfileloglog_warningpositive_numberc                   @    e Zd Zed        Zed        ZddefdZd Zy)CommandProcessc                     t               )z;Return the Popen object that refers to the current command.NotImplementedErrorselfs    g/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx/_distributed_utils/launch.pyprocesszCommandProcess.process        "##    c                     t               )z}Return a tuple (returncode, killed) for the command. It should be
        (None, None) while the command is running normally.r   r   s    r   exit_statuszCommandProcess.exit_status   s     "##r   datac                     t               )zpPreprocess the output of the command so that extra data can be
        capture or the format changed on the fly.r   )r   r   	is_stdouts      r   preprocess_outputz CommandProcess.preprocess_output%   r   r   c                     t               )z"Terminate or return the exit code.r   r   s    r   	terminatezCommandProcess.terminate*   s    !##r   NF)	__name__
__module____qualname__propertyr   r   strr!   r#    r   r   r   r      s9    $ $ $ $
$c $
$r   r   c                   `    e Zd Zd Zed        Zed        Zd	dZd Ze	d        Z
e	d        Zy)
RemoteProcessc                     |dk(  }t         j                  ||||||      }	|sd| dt        j                  |	       }	|| _        d | _        || _        t        |	ddt        t        t              | _	        d| _
        y )N	127.0.0.1zssh -tt -o LogLevel=QUIET  T	/bin/bash)shell
executablestdinstdoutstderrF)r,   make_launch_scriptshlexquote_host_pidfile	_is_localr
   r	   _process_killed)
r   rankhostpythoncwdfilesenvcommandis_localcmds
             r   __init__zRemoteProcess.__init__0   s    ;&..tS%gxX.tfAekk#6F5GHC
!"
 r   c                     | j                   S N)r<   r   s    r   r   zRemoteProcess.processD   s    }}r   c                 N    | j                   j                         | j                  fS rI   )r<   pollr=   r   s    r   r   zRemoteProcess.exit_statusH   s    }}!!#T\\11r   c                 h    | j                   %|j                  dd      ^}}|| _         |r|d   S dS |S )N
r   maxsplitr    )r:   split)r   r   r    pidfilerests        r   r!   zRemoteProcess.preprocess_outputL   s?    == !ZZqZ9NGd#DM"47**r   c                    | j                   ry | j                  j                          | j                  j                          t        j                  | j                        }| j                  s%d| j                   dt        j                  |       }t        |ddddd      }|j                  j                         dk(  | _         y )Nzssh r/   Tr0   )checkr1   r2   capture_outputtext1)r=   r<   r#   waitr,   make_kill_scriptr:   r;   r9   r7   r8   r   r4   strip)r   rF   cs      r   r#   zRemoteProcess.terminateT   s    <<! ,,T]];~~Aekk#&6%78C"
 xx~~'3.r   c                    d}|sd}|dz  }|dz  }|dz  }|xs t        j                         }|dt        |       dz  }|dt        |       d	z  }||d
z  }|dt        |       dz  }|dz  }|D ]m  }|j                  dd      ^}	}
t	        |
      dkD  rt        j                  |
d         nd}
t        d |	D              st        d| d       b|d|	 d|
 d	z  }o |j                         D ]/  \  }}|dz  }|dt        j                  |       dz  }|d| dz  }1 |d|  d	z  }|ddj                  t        t
        j                  |             dz  }|dz  }|S )NrP   zstty -echo; zpidfile=$(mktemp); zecho $$ > $pidfile; zprintf "%s\n" $pidfile; z	if [[ -d z
 ]]; then z  cd z; else z& echo 'Failed to change directory to' z >2; fi; =r   rN   r   c              3   J   K   | ]  }|j                         xs |d k(    yw)_N)isalnum).0r\   s     r   	<genexpr>z3RemoteProcess.make_launch_script.<locals>.<genexpr>   s#     <1qyy{.a3h.s   !#'z5' is an invalid environment variable so it is ignoredzexport zfname=$(mktemp); zecho z
 >$fname; z	=$fname; zexport MLX_RANK=zcmd=(r/   ); zexec "${cmd[@]}")osgetcwdreprrQ   lenr7   r8   allr   itemsjoinmap)r>   rA   rB   rC   rD   rE   scriptdekeyvalueenv_namecontents                r   r6   z RemoteProcess.make_launch_scriptj   s    #F 	''((-- 299;Id1gYj11E$q'"%%?gF>tAwiuMMF& A''#'2KC%-0Z!^EKKa)E<<<sOP uAeWB//F  "'Hg))Fekk'23:>>Fz33F "/ 	$TF"-- 	E#((3u{{G#<=>cBB$$r   c                 h    d}|d|  dz  }|dz  }|dz  }|dz  }|dz  }|dz  }|d	z  }|d
|  z  }|S )NrP   z
pid=$(cat rg   zif ps -p $pid >/dev/null; then z    kill $pid; z    echo 1; r^   z    echo 0; r_   zrm r*   )rR   rp   s     r   rZ   zRemoteProcess.make_kill_script   sm    Jwis++33##. '. &Cy/!r   Nr$   )r%   r&   r'   rG   r(   r   r   r!   r#   staticmethodr6   rZ   r*   r   r   r,   r,   /   sb    (   2 2/, . .`  r   r,   c           
      T	    ddgt        |      z   fd}g }g }g }g }t        |      D ]  \  }\  }	}
|j                  t                      |j                  t                      |j                  t                      t	        j
                  ||	|
|d   |d   |d   dz        }|j                          |j                  |        t        j                  t        j                  j                         d       t        j                  t        j                  j                         d       t        j                  t        j                  j                         d       rt        d t        ||      D              rt!        t        j                  j                         gg g d	      \  }}}|D ]0  }t        j"                  |d
      }|D ]  }|j%                  |        2 |D ][  }	 |j'                         sHt        j                  j(                  j+                  |j-                                |j'                         sH] |D ][  }	 |j'                         sHt        j                  j(                  j+                  |j-                                |j'                         sH] t        j                  j(                  j1                          t        j                  j(                  j1                          t        d |D              r4t        |      D ]%  \  }}|j3                         r|   d   dk7  s#d n nn"st        d t        ||      D              r|D ]  }|j5                           |D ][  }|j'                         rt        j                  j(                  j+                  |j7                                |j'                         sH] |D ][  }|j'                         rt        j                  j(                  j+                  |j7                                |j'                         sH] t        j                  j(                  j1                          t        j                  j(                  j1                          y # t.        $ r Y w xY w# t.        $ r Y [w xY w)NF)NNc                 z   |j                  d      }|j                  d      }|j                  d      } | g|i |}|j                  }t        j                  |j                  j                         d       t        j                  |j                  j                         d       t        j                  |j                  j                         d       |j                  j                         |j                  j                         g}|j                  j                         g}	d}
|j                         	 |
|j                         z  }
t        ||	g d      \  }}}|D ]  }||j                  j                         k(  }t        j                  |d      j                  d	      }|j                  ||      }|r |j                  |j!                                z|j                  |j!                                 |D ],  }t#        |
      d
kD  st        j$                  ||
      }|
|d  }
. r|j'                          n|j                         |j(                  | <   |    d   rt+        d|  d       y |    d
   d
k7  rt+        d|  d|    d
           y t-        d|  d       y # t        $ r Y lw xY w)Nstdin_queuestdout_queuestderr_queueFr   g      ?    ignore)errorsr   r   zNode with rank z was killedz exited with code z
 completed)popr   rh   set_blockingr4   filenor5   r3   rK   
get_nowait
QueueEmptyr   readdecoder!   putencoderk   writer#   r   r   r   )r>   argskwargsr{   r|   r}   rD   pto_readto_writestdin_bufferrlistwlistrb   fdr    msgncommand_class
exit_codesstopverboses                     r   
_thread_fnz#_launch_with_io.<locals>._thread_fn   se   jj/zz.1zz.16t6v6OO
)51
)51
(%088??$ahhoo&78GGNN$%ffh 6 6 88 %WhC@OE5!!((//"33	ggb$'..h.?//Y? $$SZZ\2 $$SZZ\2  |$q(\2A#/#3L  !!#) ffh* #..
4dA/${;<a A%/$/A*TBRSTBUAVWX?4&
;<3  s   :J- -	J:9J:)r{   r|   r}   )targetr   r   Tc              3   >   K   | ]  }|j                            y wrI   )empty)rd   qs     r   re   z"_launch_with_io.<locals>.<genexpr>   s     U1TAaggi-1Ts   g?r~   c              3   <   K   | ]  }|j                           y wrI   )is_alive)rd   ts     r   re   z"_launch_with_io.<locals>.<genexpr>  s     -Wqzz|Ws   r   )rk   	enumerateappendr   	threadingThreadstartrh   r   sysr3   r   r4   r5   anyr   r   r   r   r   bufferr   r   r   flushr   rn   get)r   	argumentsr   r   stdin_queuesstdout_queuesstderr_queuesthreadsir   r   r   r   rb   r   r   r   r   r   s   ` `              @@r   _launch_with_ior      s   D#i.0J+=Z LMMG&y1>D&EG$UW%UW%+B/ -b 1 -b 1	
 	
	q 2" OOCII$$&.OOCJJ%%'.OOCJJ%%'.cU}m1TUUcii..012r3?q!B772t,L!l# "  A'')JJ%%++ALLN; '')  A'')JJ%%++ALLN; '')  	

!

! -W--!'*1zz|!!}Q'1,#	 + A cU}m1TUUF 	  '')JJ##AEEG, '')  '')JJ##AEEG, '')  JJJJC    s&   /AR
AR
	RR	R'&R'c                    t        d |D              r| j                  d       |j                  }g }|D ][  }g }|j                  D ]7  }t	        |j
                        D ]  }	|j                  | d|        |dz  } 9 |j                  |       ] t        |      dkD  rt        j                  |      nd}
d|
i}|j                  }|j                  r|j                  d       |j                  }t        |j                  dt        j                  |             t!        t"        t%        |      D cg c]$  \  }}||j&                  |j(                  ||||fi f& c}}|j                         y c c}}w )	Nc              3   L   K   | ]  }t        |j                        d k(    yw)r   N)rk   ipsrd   hs     r   re   zlaunch_ring.<locals>.<genexpr>%  s     
*Eq3quu:?Es   "$zAThe ring backend requires IPs to be provided instead of hostnames:r   rP   MLX_HOSTFILEzMLX_RING_VERBOSE=1Running)r   errorstarting_portr   rangeconnections_per_ipr   rk   jsondumpsrC   r   rA   r   r7   rn   r   r,   r   ssh_hostnamer@   )parserhostsr   rD   port
ring_hostsr   nodeipr   hostfilerB   rC   rA   r>   s                  r   launch_ringr   $  sS   

*E
**O	
 DJ%%B4223rd!D6N+	 4  	$  *-Z1)<tzz*%"HX&E
((C||

'(
((CiG!45 %U+	
+a ANNDKKeS'JBO+	
 		
s   2)E/c                    |d   j                   st        d      |d   j                   d   }|j                  }t        |      }|j                  }|j
                  }|j                  r|j                  d       |j                  d|        |j                  d|        |j                  d|        t        |j                  dt        j                  |             t        t        t        |      D 	
cg c]8  \  }	}
|	|
j                  |j                  |i |d|	|j                   z   gz   |fi f: c}
}	|j                         y c c}
}	w )	Nr   7Rank 0 should have an IP reachable from all other rankszNCCL_DEBUG=INFOzNCCL_HOST_IP=z
NCCL_PORT=zMLX_WORLD_SIZE=r   zCUDA_VISIBLE_DEVICES=)r   
ValueError	nccl_portrk   rC   rA   r   r   r   r7   rn   r   r,   r   r   r@   repeat_hosts)r   r   r   rD   master_hostmaster_port
world_sizerC   rA   r>   r   s              r   launch_ncclr   G  s<   8<<RSS(,,q/K..KUJ
((C
((C||

$%JJ{m,-JJK=)*JJ-.iG!45 %U+	
 ,a NNKK24$:K:K3K2LMNN  ,	
 	#	
s   7=Ec                     d   j                   st        d      |j                  dk(  }t        fdD              }t        d t	              D              }|r|st        d      d   j                   d   }|j
                  }|j                  }	|j                  d| d|j                          |r|j                  d	       d
t        j                  D 
cg c]  }
|
j                   c}
      i}t        |j                  dt        j                  |             t!        t"        t	              D 
cg c]$  \  }}
||
j$                  |j&                  |	|||fi f& c}
}|j                         y c c}
w c c}
}w )Nr   r   
jaccl-ringc              3   `   K   | ]%  }t        |j                        t              k(   ' y wrI   )rk   rdma)rd   r   r   s     r   re   zlaunch_jaccl.<locals>.<genexpr>s  s#     >1S[CJ.s   +.c              3   D   K   | ]  \  }}|j                   |   d u   y wrI   )r   )rd   r   r   s      r   re   zlaunch_jaccl.<locals>.<genexpr>t  s$     D3C41aQVVAY$&3Cs    z$Malformed hostfile for jaccl backendzMLX_JACCL_COORDINATOR=r   zMLX_JACCL_RING=1MLX_IBV_DEVICESr   )r   r   backendrl   r   rC   rA   r   r   r   r   r   r   r   r7   rn   r   r,   r   r@   )r   r   r   rD   
jaccl_ring
have_rdmas
have_nullscoordinatorrC   rA   r   rB   r>   s    `           r   launch_jacclr   n  sR   8<<RSS-J>>>JD9U3CDDJZ?@@(,,q/K
((C
((CJJ'}Ad6H6H5IJK

%&

E+BEqAFFE+B CDEiG!45 %U+	
+a ANNDKKeS'JBO+	
 		 ,C	
s   E5
8)E:c                     	 t        ddgdd      } | j                  j                         j                         } t        j                         dk(  rt        dd| gdd      }nt        d| gdd      }|j                  j                         }t        t        d	 |j                                     }|j                         j                         d
   j                  d      S #  Y y xY w)Nwhich	ompi_infoTrU   rV   Darwinotoolz-Llddc                 
    d| v S )Nlibmpir*   )lines    r   <lambda>z!get_mpi_libname.<locals>.<lambda>  s	    D 0r   r   z@rpath/)r   r4   r[   r   platformsystemnextfilter
splitlinesrQ   removeprefix)r   otool_outputlibmpi_lines      r   get_mpi_libnamer     s    +.d4P	$$**,335	??($	*$tL y1dSL#**113 0,2I2I2KL
   "((*1-::9EEs   CC! !C%c           
         t        ddgdd      }|j                  j                         j                         }t	               }|Gt        |      j                  j                  dz  }dt        |       d| g|j                  z   |_        t        |j                  d| d	       t        j                  d
      5 }t        d |D              }|j                         D ]  \  }}	t        | d|	 |        |j!                          |ddd|j"                  g|j$                  rd|j$                  gng t'        d |j                  D        g       t'        |j(                  D 
cg c]  }
t+        j,                  |
       c}
g       d|}t        |j                  ddj/                  |             	 t        |       d d d        y c c}
w # t0        $ r Y w xY w# 1 sw Y   y xY w)Nr   mpirunTr   libzDYLD_LIBRARY_PATH=zMLX_MPI_LIBNAME=zUsing 'rf   w)modec              3   4   K   | ]  }|j                     y wrI   )r   r   s     r   re   zlaunch_mpi.<locals>.<genexpr>  s     7As   z slots=)filez--outputz:raw
--hostfilez-cwdc              3   $   K   | ]  }d |g 
 yw)z-xNr*   )rd   rr   s     r   re   zlaunch_mpi.<locals>.<genexpr>  s     .X4)Xs   --r   r/   )r   r4   r[   r   r   r   parentr)   rC   r   r   tempfileNamedTemporaryFiler   rm   printr   namerA   summpi_argr7   rQ   rn   KeyboardInterrupt)r   r   r   rD   r   mpi_libnamedyldfr   r   argrF   s               r   
launch_mpir
    s   '8$DFF]]  "))+F "#KF|""))E1 T,{m,
 HH
 xq)*		$	$#	.!778KKMDAqQCwqc"+ "		 FF
 %)HHvtxx "
 .TXX.3
 dll;ls%++c"l;R@
 
 
 	DLL)SXXc]3	H) 
/	. < ! 		+ 
/	.s=   <B)G%G	3G5G	G	GGGGG&c                     t        j                  d      } | j                  ddd       | j                  ddd       | j                  d	d
d       | j                  ddt        dd       | j                  dd       | j                  dd       | j                  ddg d       | j                  ddg d       | j                  ddt        d       | j                  d d!t        d"d#       | j                  d$d%       | j                  d&t        d'd(       | j                  d)d*d+d,-       | j                  d.t
        j                  d/       | j                         \  }}|j                  rt        |j                         y t        |      d0k(  r| j                  d1       |d0   d2k(  r|j                  d0       |j                   t        j                   |j                        }n*t        j"                  |j$                  |j&                        }|j(                  d3k7  r|j(                  |j(                  |_        |j(                  't*        j,                  j/                         rd4nd5|_        |j0                  |j2                  z   |_        t5        |d0         x}j7                         r9|j9                         r)|j                  t;        |j=                               g|d0d n=t?        j@                  |d0         x}||d0<   n|jB                  rtE        d6|d0          |j(                  d5k(  rtG        | |j$                  ||       y |j(                  d7k(  rtI        | |j$                  ||       y |j(                  d4k(  rtK        | |j$                  ||       y |j(                  d8k(  s|j(                  d9k(  rtM        | |j$                  ||       y | j                  d:       y );Nz!Launch an MLX distributed program)descriptionz--print-python
store_truez8Print the path to the current python executable and exit)actionhelpz	--verbosezPrint debug messages in stdoutz--hostsr.   zA comma separated list of hosts)defaultr  z--repeat-hostsz-nr   z(Repeat each host a given number of times)typer  r  r   zThe file containing the hosts)r  z	--backendz#Which distributed backend to launchz--envr   z&Set environment variables for the jobs)r  r  r  z	--mpi-argz$Arguments to pass directly to mpirunz--connections-per-ipz7How many connections per ip to use for the ring backend)r  r  r  z--starting-portz-piC~  zHFor the ring backend listen on this port increasing by 1 per rank and IPz--cwdz:Set the working directory on each node to the provided onez--nccl-porti90  zBThe port to use for the NCCL communication (only for nccl backend)z--no-verify-scriptstore_falseverify_scriptz$Do not verify that the script exists)r  destr  z--pythonz#Use this python on the remote hostsr   zNo script is providedr   rP   ncclringzInvalid script or command mpijacclr   zKThe backend should be one of {'ring', 'mpi', 'nccl', 'jaccl', 'jaccl-ring'})'argparseArgumentParseradd_argumentr   intr   r2   parse_known_argsprint_pythonr  r@   rk   r   r   r   r   	from_file	from_listr   r   r   mxcudais_availableenvsrC   r   existsis_filer)   resolveshutilr   r  r   r   r
  r   r   )r   r   rS   r   rp   rD   s         r   mainr)    s   $$1TUF
G  
 L/O   ;-N   7   +JK
2   5	   3	   F	   W   R   Q	   3	   CNN1V   ((*JD$dkk
4yA~,-Aw$ }} %%dmm4%%djj$2C2CD 2$,,"6''||!#!5!5!7vV}}txx'DH tAw-'')fnn.>[[#fnn&6"78Qq	\\$q'*
*'	7Q			5d1gY?@@ ||vFHNND$7		68>>46		FHNND$7		 DLLL$@VX^^T48Y	
r   ))r  base64r   rh   r7   r(  r   r   r   collectionsr   	itertoolsr   pathlibr   queuer   r   r   r   
subprocessr	   r
   r   mlx.corecorer!  commonr   r   r   r   r   r,   r   r   r   r   r   r
  r)  r*   r   r   <module>r3     s       	   
      %   ' '  ? ?$ $,xN xvwt F$N<,$Nq
r   