
    (HJjf0              	       0   d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZmZmZmZm Z  d
 Z!ejD                  ddfdejF                  de$de%fdZ&	 dde'de(de(de(fdZ)d Z*y)    N)Path)tree_map)tqdm)load_dataset)kl_div_loss)grad_checkpointiterate_batches)print_trainable_parameters)loadload_tokenizerpipeline_loadquantize_modelsavec                      t         j                  j                         j                          fd} |||d        |||d       y )Nc           
          dk(  r||z  }|j                  dd       t        t        t        | 
            t	        |       
z  d| dk7        x}D ]  \  }\  }}|d d d df   } |      }t        j                  |t
        j                        }t        j                  |       dk(  s\t        j                  |d	d
      dd	d f   }t        j                  ||d      }||ddz  }	t        j                  |	||d        y )Nr   T)parentsexist_okseedzComputing targets for )totaldescdisablestreami )kthaxis.r   010d.safetensors)logitsindices)mkdirr   	enumerater	   lenmxstop_gradientcpuevalargpartitiontake_along_axissave_safetensors)datapathsplitpbaribatch_r!   idxfile
batch_sizemax_seq_lengthmodelrankr   s             Z/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/quant/dwq.py_compute_targetsz-compute_dwq_targets.<locals>._compute_targets(   s   19%<DJJtdJ3/$
NQUVW$i:--eW5		 D  Azq !SbS&ME5\F%%fRVV<FGGFOqyoof%bA#uv+N++FCbA455##DV*LM#    validtrain)r&   distributedinitr9   )	r8   save_dir
train_data
valid_datar6   r7   r   r;   r9   s	   `   ``` @r:   compute_dwq_targetsrD      sF     >> %%'DN N2 Z73Z73r<   Fg       @dtypegradient_checkpointtemperaturec                     !"#$ t         j                  j                         }|j                         $|j	                         !!fd"d } j                           j                  |       t                |	rt         j                  d          d|
z  # #fd  fd} "$fd}t        d  j                               }d	}d}d}t        j                         } ||d
      x}}t        t        t        |            t!        |      z        x}D ]  \  }\  }}|d d d df   } ||d      }t        j"                  |        |||||      \  }}}t        j"                  ||       t         j                  j%                  |t         j&                        j)                         $z  }t         j                  j%                  |t         j&                        j)                         }||z  }|||z  z  }!dk(  r|j+                  d|d       |dz   dz  dk(  rm|t        j                         |z
  z  }t        j,                         dz  }||z  }||z  } "d|d|dd|d|dd|d
       t        j                         }d}d}|dz   dz  dk(  s |||
      }  ||
      }||k  r "d|dd|dd        j/                  t        fd |             y )!Nc                  <    dk(  rt        j                  | i | y y )Nr   )r   write)argskwargsr9   s     r:   rprintzdwq_quantize.<locals>.rprintV   s     19JJ'' r<   c                     t        |d      rCt        |d      r6|j                  dk(  r&|j                  dk  r|j                  ddgd       y y y y y )	Nbits
group_sizeaffine   scalesbiasesF)keysrecurse)hasattrmoderO   unfreeze)r3   ms     r:   rY   zdwq_quantize.<locals>.unfreezeZ   sV    Av<(("
JJXx0%J@  # ) r<   r      c                    j                  t        
fd|               |      }t        |t              r|\  }}t	        j
                  ||d      }t        |z  |z        }t	        j                  dd|j                  d   z         |d d dd f   k  }|j                         }||z  j                         |z  }	|	|fS )Nc                 &    | j                        S NastypexrE   s    r:   <lambda>z/dwq_quantize.<locals>.loss_fn.<locals>.<lambda>m   s    r<   r   r   r[   )
updater   
isinstancetupler&   r+   r   arangeshapesum)paramsrb   targetslengthsr!   idslossesmaskntokslossrE   r8   scales             r:   loss_fnzdwq_quantize.<locals>.loss_fnl   s    X7@Aqgu%"LGS''"=FUV^UW_=yyAa 001GAqrENB
v""$u,U{r<   c                      t        j                        || ||      \  \  }}}t        j                  |      }j	                  ||      }|||fS r^   )r&   value_and_gradnnaverage_gradientsapply_gradients)	inputsrk   rl   rj   rq   rp   gradsrs   opts	          r:   stepzdwq_quantize.<locals>.stepx   s]    9r009FGW 
uu $$U+$$UF3UF""r<   c           
      b   d}d}t        t        t        
            t              
z  dd      D ]  \  }\  }}|d d d df   } ||d	      }t	        j
                  |        | |||      \  }}	t	        j
                  ||	       t        j                  j                  |t        j                  
      j                         z  }t        j                  j                  |	t        j                  
      j                         }	||	z  }|||	z  z  } ||z  } d|d|d       |S )N        r   r   zComputing validation lossF)r   r   leaver   r=   r/   r   zValidation: it=z, loss=.3f)
r   r$   r	   r%   r&   r)   r?   all_sumr(   item)rj   itv_lossv_tokensr1   r2   rl   rk   rq   rp   r6   rs   r7   rM   r   	target_fnrC   
world_sizes             r:   validatezdwq_quantize.<locals>.validate   s1   #'
JTR j/Z/,$
Aw !SbS&MEq8GGGG!&%'BKD%GGD% >>))$rvv)>CCE
RDNN**5*@EEGEHdUl"F!$
"  !bU(T3K01r<   c                 @    | j                  t        j                        S r^   )r`   r&   float32)rb   s    r:   rc   zdwq_quantize.<locals>.<lambda>   s    !((2::&r<   r~   )r   r   )r   r   r>   r   r   zloss=z.4f)r      g    eAzit=z, avg_loss=z, total_tokens=z, toks_per_sec=r   z, peak_memory_gb=   u*   ❌❌❌
[WARNING] Final validation loss z' is worse than initial validation loss u2   . Model quality will likely be degraded.
❌❌❌c                 &    | j                        S r^   r_   ra   s    r:   rc   zdwq_quantize.<locals>.<lambda>   s    AHHUOr<   )r&   r?   r@   sizer9   r>   apply_to_modulesr
   r   layersr   trainable_parameterstimer   r$   r	   r%   r)   r   r(   r   set_descriptionget_peak_memoryrd   )%r8   r   r{   rB   rC   r6   r7   r   rE   rF   rG   grouprY   r|   r   rj   
total_losstotal_tokenstokensticinitial_valid_loss
valid_lossr0   r   r2   rl   rk   rq   rp   toks_per_secpeak_memory_gbavg_lossrs   r9   rM   rr   r   s%   ``` `````                       @@@@@r:   dwq_quantizer   E   s    NN!EJ::<D(A 
KKM	8$u%Q(OE
# 2 &""$F
 JLF
))+C '/v!&<< 
JTR j/Z/	
 	
 	
 	UG a"fE2W5
"5'7FCeV
f~~%%d266%:??AJN&&uRVV&<AAC%dUl"
19  sn 5Q"}!%s):;!#!3!3!5!;%.&re<h_,<|o >&$c**<^S,AC iik
Fc>Q!&R0J?	
B &R(JJ&9*S9I J22DS1I JAA	
 
LL3V<=r<   	data_pathnum_samplesr7   num_valid_samplesc                 z   t        j                  |ddddd      }t        ||       d   t        j                  j                  t                    }|d | j                         }||||z    j                         }fd}	|D 
cg c]
  }
 |	|
       }}
|D 
cg c]
  }
 |	|
       }}
||fS c c}
w c c}
w )	Nr>   z	train[:1])r.   train_splitvalid_splitTF)
hf_datasetr>   testr   c                 @    j                  |          \  }}|d  |fS r^   )process)r4   r   offsetdatasetr7   s      r:   r   zload_data.<locals>.process   s+     6'00r<   )typesSimpleNamespacer   nprandompermutationr%   tolist)	tokenizerr   r   r7   r   rK   perm
train_perm
valid_permr   r1   r>   r=   r   s      `         @r:   	load_datar      s       "&

 D 4+A.G99  W.Dl{#**,JkK2C$CDKKMJ1 ",,AWQZE,!+,AWQZE,%< -,s   	B3B8c                  
   t        j                         } | j                  dddt        d       | j                  dt        d d       | j                  d	d
d       | j                  dt        dd       | j                  dt        dd       | j                  dt        dd       | j                  dt        d       | j                  dt        d       | j                  dt
        d       | j                  dt        d       | j                  dt        dd        | j                  d!d"d#$       | j                  d%t        d d&       | j                  d'd"d($       | j                  d)d"d*$       | j                         }t        j                  j                         }|j                  }|j                  s=||j                         z  d+kD  r'||j                         ||j                         z  z
  z  }t        j                  j                  |j                         t        j                  j                  |j                         |j                   dt#        |j                         j%                         xr< t'        d,z  j)                  d-            xr t'        d.z  j)                  d-            }nd/}d t+        |j,                        }t/        ||j0                  |j                  |j2                        \  }}|r|j4                  X|j                  r/|j                         d0kD  rt7        |j,                  d1      \  }}	nt9        |j,                  dd2      \  }}	nd |s42t;        |||j<                  |j2                  |j                  3       d}|j>                  rtA        d+       |rfd4}
nfd5}
|j4                  +t9        |j4                  dd6      \  }}}	d7|	vrFtC        d8      tE        jF                        }tI        |	|jJ                  |jL                  9      \  }}	|rt        jN                  jQ                         r,t        jR                         d:   }t        jT                  |       tW        jX                  |jZ                  d;      }t]        ||
||||j<                  |j2                  |j                  |j^                  <	       ta        |jb                  |j,                  |||	       y )=Nz--modelz-mzA model to distill from for DWQ. If `quantized-model` is not given the student model will be this model quantized according to `bits` and `group-size`.T)helptyperequiredz--quantized-modelzCAn already quantized model (the student model) to improve with DWQ.)r   defaultr   z
--mlx-path	mlx_modelz!Path to save the quantized model.)r   r   z--bits   z!Bits per weight for quantization.z--group-size@   zGroup size for quantization.z--num-samplesi   z&Number of samples to use for training.z--max-seq-lengthi  )r   r   z--seed{   z--learning-rategư>z--batch-sizez--data-pathzallenai/tulu-3-sft-mixturezIA Hugging Face dataset which is compatible with an mlx-lm dataset format.z--grad-checkpoint
store_truez0Use gradient checkpointing to reduce memory use.)actionr   z--target-dirzDirectory to save/load targets.z--targets-onlyzCompute the targets and exit.z
--pipelinez/Use pipeline parallel instead of data parallel.r   r>   z*.safetensorsr=   Fr[   )return_config)r   lazy)r6   r7   r   c                 V    t        j                  |z  |ddz        }|d   |d   fS )Nr   r    r!   r"   )r&   r   )r3   r4   r/   rk   
target_dirs       r:   r   zmain.<locals>.target_fng  s9    ggj50c$Z|3LLMG8$gi&888r<   c                      |       S r^    )r2   r4   r/   r8   s      r:   r   zmain.<locals>.target_fnm  s    <r<   )r   r   quantizationz*Quantized model must already be quantized.)rP   rO    max_recommended_working_set_size)learning_ratebias_correction)r6   r7   r   rF   )2argparseArgumentParseradd_argumentstrintfloat
parse_argsr&   r?   r@   r   pipeliner   r   r   r   r   r   is_diranyglobr   r8   r   r   r7   quantized_modelr   r   rD   r6   targets_onlyexit
ValueErrorcopydeepcopyr   rP   rO   metalis_availabledevice_infoset_wired_limit
optimizersAdamr   r   r   r   mlx_path)parserrK   r   r   has_targetsr   rB   rC   r3   configr   q_modelmax_rec_sizer{   r8   r   s                 @@r:   mainr      s   $$&F
'    R	   k0S   0	   S"3Q   5	   *dC
sC8
)tD
S!<
,X	   ?  
 S$5V   4S   >   DNN!E""K==[5::<7!;uzz|kEJJL&@@@IINN499IINN499"$//*
 BZ')//@ABZ')//@A 	 
tzz*I&4>>4#3#3T5H5HJ

 $..6==UZZ\A-,TZZtLE1f#DJJdNE1f :1..	
 Q	9	  '%)  &
"F
 'IJJ--&"	
	6 u(	xx~~'(JK
<(
//(:(:D
QC??**YY 00
 	

r<   )    )+r   r   r   r   pathlibr   mlx.corecorer&   mlx.nnrv   mlx.optimizersr   numpyr   	mlx.utilsr   r   mlx_lm.tuner.datasetsr   mlx_lm.tuner.lossesr   mlx_lm.tuner.trainerr   r	   mlx_lm.tuner.utilsr
   mlx_lm.utilsr   r   r   r   r   rD   bfloat16Dtypeboolr   r   r   r   r   r   r   r<   r:   <module>r      s           #    . + A 9 %4b kk %L> 88L> L> L>h    	
 <ir<   