
    (HJjt%                         d dl Z d dlZd dlmZ d dlmZmZmZ d dlm	Z	  G d d      Z
 G d d      Z G d	 d
      Z G d d      Z G d d      Zde	fdZdede	fdZdede	fdZde	fdZde	fdZy)    N)Path)AnyDictList)PreTrainedTokenizerc                   P    e Zd ZdZ	 ddeeeef      dedefdZd Z	de
fdZd	 Zy
)TextDatasetz1
    Light-weight wrapper to hold a dataset.
    data	tokenizertext_keyc                 .    || _         || _        || _        y N)_datar   r   )selfr
   r   r   s       _/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/tuner/datasets.py__init__zTextDataset.__init__   s     
"     c                     | j                   j                  || j                           }|d   | j                   j                  k7  r%|j	                  | j                   j                         |dfS )Nr   )r   encoder   eos_token_idappendr   ds     r   processzTextDataset.process   sT    NN!!!DMM"23R5DNN///HHT^^0011vr   idxc                      | j                   |   S r   r   r   r   s     r   __getitem__zTextDataset.__getitem__        zz#r   c                 ,    t        | j                        S r   lenr   r   s    r   __len__zTextDataset.__len__#       4::r   N)text)__name__
__module____qualname____doc__r   r   strr   r   r   intr    r&    r   r   r	   r	      sP     	!4S>"! '! 	!s r   r	   c            	       V    e Zd ZdZ	 	 ddeeeef      dededefdZ	d Z
defd	Zd
 Zy)ChatDatasetz
    A dataset for chat data in the format of {"messages": [...]}
    https://platform.openai.com/docs/guides/supervised-fine-tuning#formatting-your-data
    r
   r   chat_keymask_promptc                 <    || _         || _        || _        || _        y r   )r   r2   r3   r   )r   r
   r   r2   r3   s        r   r   zChatDataset.__init__-   s!     
 &"r   c                 ,   || j                      }|j                  dd       }| j                  j                  ||d      }| j                  rF|d   j                  d      dk(  }t        | j                  j                  |d d ||d            }||fS |dfS )	NtoolsFr6   return_dictr   role	assistantr6   add_generation_promptr8   r   )r2   getr   apply_chat_templater3   r$   )r   r   messagesr6   tokensr<   offsets          r   r   zChatDataset.process9   s    T]]#gt$33 4 

 $,RL$4$4V$<$K!22SbM*? %	 3 F F##A;r   r   c                      | j                   |   S r   r   r   s     r   r    zChatDataset.__getitem__O   r!   r   c                 ,    t        | j                        S r   r#   r%   s    r   r&   zChatDataset.__len__R   r'   r   N)r?   Fr)   r*   r+   r,   r   r   r-   r   boolr   r   r.   r    r&   r/   r   r   r1   r1   '   s]     #!
#4S>"
# '
# 	
#
 
#,s r   r1   c            
       T    e Zd ZdZdeeeef      dedededef
dZ	d Z
d	efd
Zd Zy)CompletionsDatasetz
    A dataset for prompt-completion data in the format of {"prompt": ..., "completion": ...}
    or using user-provided keys for prompt and completion values
    https://platform.openai.com/docs/guides/fine-tuning/example-format
    r
   r   
prompt_keycompletion_keyr3   c                 J    || _         || _        || _        || _        || _        y r   )r   rH   rI   r3   r   )r   r
   r   rH   rI   r3   s         r   r   zCompletionsDataset.__init__]   s)     
$,&"r   c                 (   |j                  dd       }d|| j                     dd|| j                     dg}| j                  j	                  ||d      }| j
                  r/t        | j                  j	                  |d d |dd	            }||fS |d
fS )Nr6   user)r9   contentr:   Fr7   r   Tr;   r   )r=   rH   rI   r   r>   r3   r$   )r   r   r6   r?   r@   rA   s         r   r   zCompletionsDataset.processk   s    gt$$//(:; Qt/B/B-CD
 33Eu 4 
 22SbM*. %	 3 F F##{r   r   c                      | j                   |   S r   r   r   s     r   r    zCompletionsDataset.__getitem__   r!   r   c                 ,    t        | j                        S r   r#   r%   s    r   r&   zCompletionsDataset.__len__   r'   r   NrD   r/   r   r   rG   rG   V   s_    #4S>"# '# 	#
 # #,s r   rG   c                   6    e Zd Zdee   fdZdefdZd Zd Z	y)ConcatenatedDatasetr
   c                 T    || _         t        d | j                   D              | _        y )Nc              3   2   K   | ]  }t        |        y wr   )r$   ).0r   s     r   	<genexpr>z/ConcatenatedDataset.__init__.<locals>.<genexpr>   s     3
1A
s   )r   sum_lenr   r
   s     r   r   zConcatenatedDataset.__init__   s    
3

33	r   r   c                     t        | j                        D ]  \  }}|t        |      z
  }|dk  r n|} |   }|d<   |S )Nr   _dataset)	enumerater   r$   )r   r   data_idxr
   jdatums         r   r    zConcatenatedDataset.__getitem__   sO    '

3NHdc$iA1uC	 4
 S	$jr   c                 D    | j                   |d      j                  |      S )NrZ   )r   r   r   s     r   r   zConcatenatedDataset.process   s     zz!J-(0033r   c                     | j                   S r   )rW   r%   s    r   r&   zConcatenatedDataset.__len__   s    yyr   N)
r)   r*   r+   r   r   r   r.   r    r   r&   r/   r   r   rQ   rQ      s(    4T#Y 4s 4r   rQ   c                   6    e Zd ZdefdZdefdZdefdZd Zy)CacheDatasetr
   c                 :    || _         d gt        |      z  | _        y r   )r   r$   
_proc_datarX   s     r   r   zCacheDataset.__init__   s    
&3t9,r   r   c                 2    t        | j                  |         S r   r#   r   s     r   itemlenzCacheDataset.itemlen   s    4::c?##r   c                     | j                   |   5| j                  j                  | j                  |         | j                   |<   | j                   |   S r   )rd   r   r   r   s     r   r    zCacheDataset.__getitem__   sE    ??3'#'::#5#5djjo#FDOOC s##r   c                 ,    t        | j                        S r   r#   r%   s    r   r&   zCacheDataset.__len__   r'   r   N)	r)   r*   r+   r   r   r.   rf   r    r&   r/   r   r   rb   rb      s+    -S -$3 $$s $
r   rb   r   c                 6   t        |dd      }t        |dd      }t        |dd      }t        |dd      }t        |d	d
      }| d   }||v r||v rt        | ||||      S ||v rt        | |||      S ||v r|rt        d      t	        | ||      S t        d      )Nr3   Fprompt_featureprompttext_featurer(   completion_feature
completionchat_featurer?   r   )r2   r3   z.Prompt masking not supported for text dataset.)r   z~Unsupported data format, check the supported formats here:
https://github.com/ml-explore/mlx-lm/blob/main/mlx_lm/LORA.md#Data.)getattrrG   r1   
ValueErrorr	   )	r
   r   configr3   rj   rl   rm   ro   samples	            r   create_datasetrt      s    
 &-7KV%5x@N6>6:L )=|L6>:>L!WF$6&$@!)^-?
 	
 
	)l
 	
 
	MNN4\BBR
 	
r   	data_pathc                 j    fd}d}|D cg c]  } || | dz         c}\  }}}|||fS c c}w )Nc                     | j                         sg S t        | d      5 }|D cg c]  }t        j                  |       }}d d d        t	              S c c}w # 1 sw Y   xY w)Nr)existsopenjsonloadsrt   )pathfidlr
   rr   r   s       r   load_subsetz'load_local_dataset.<locals>.load_subset   sY    {{}I$_+./3aDJJqM3D/ dIv66 0 _s   AAAAA&trainvalidtestz.jsonlr/   )	ru   r   rr   r   namesnr   r   r   s	    ``      r   load_local_datasetr      sM    
7 'EINOA+iQCv,&>?OE5$% Ps   0data_idc                     ddl m}m} 	  ||       }d}|D cg c]&  }||j                         v rt	        ||   ||      ng ( c}\  }}	}
||	|
fS c c}w # |j
                  $ r t        d|  d      w xY w)Nr   )
exceptionsload_datasetr   z Not found Hugging Face dataset: z .)datasetsr   r   keysrt   DatasetNotFoundErrorrq   )r   r   rr   r   r   datasetr   r   r   r   r   s              r   load_hf_datasetr      s    
 2Iw'* 
  & wqz9f= 
ud %
 ** I;G9BGHHIs   A +AA A A3c                 x   dd l fd}| j                  }t        |t              r|g}g }|D ]  }|d   }t	        d| d       t        | dd      |d<   t        j                  di |}|j                  di       }| j                  r;|j                  d	d
      }	|j                  dd      }
 ||||	|      } ||||
|      }ng g }}| j                  r|j                  d      } |||||      }ng }|j                  |||f        t        |      dk(  r|d   S t        t        t        t!        |             S )Nr   c                 H     j                   | fd|i|}t        ||      S )Nsplit)r   rt   )dataset_namerr   r   	hf_configdsr   r   s        r   create_hf_datasetz1load_custom_hf_dataset.<locals>.create_hf_dataset   s;    "X""

 

 b)V44r   r}   Loading Hugging Face dataset .r3   Frr   train_splitztrain[:80%]valid_splitztrain[-10%:]
test_split   r/   )r   
hf_dataset
isinstancedictprintrp   typesSimpleNamespacer=   r   r   r   r$   tuplemaprQ   zip)argsr   r   dataset_collection
collectionr   ds_pathrr   r   r   r   r   r   r   r   r   s    `             @r   load_custom_hf_datasetr      sf   5 $d+01J V*-gYa89#D-?=&&,,FF8R(	::&&>K&&?K%	E &	E r5E99-J$	D D5%./G !J :!!} (#z*:;<<r   c                    t        | dd      rt        | |      \  }}}nkt        | j                        }|j	                         rt        |||       \  }}}n4t        d| j                   d       t        | j                  ||       \  }}}| j                  rt        |      dk(  rt        d      | j                  rt        |      dk(  rt        d       | j                  rt        |      dk(  rt        d      |||fS )	Nr   Fr   r   r   zKTraining set not found or empty. Must provide training set for fine-tuning.zUWarning: Validation set not found or empty. Training will proceed without validation.zBTest set not found or empty. Must provide test set for evaluation.)rp   r   r   r
   ry   r   r   r   r   r$   rq   r   )r   r   r   r   r   ru   s         r   r   r   5  s    t\5)3D)DudO	!3Iy$!OE5$1$))A>?!0It!LE5$zzc%jAoY
 	
 zzc%jAoc	
 yySY!^P
 	
 %r   )r{   r   pathlibr   typingr   r   r   transformersr   r	   r1   rG   rQ   rb   rt   r   r-   r   r   r   r/   r   r   <module>r      s       " " , 8, ,^/ /d , "
"
<"""69=,? 9=x"5 r   