ó
    …~i°  ã                   ó†   • S SK JrJrJrJrJrJr  S SKJrJ	r	J
r
JrJrJr  S SKJr  S SKJrJrJr  SSKJr   " S S\5      rg	)
é    )ÚDictÚIteratorÚListÚOptionalÚTupleÚUnion)Ú
AddedTokenÚ	TokenizerÚdecodersÚpre_tokenizersÚ
processorsÚtrainers)ÚBPE)Ú	LowercaseÚSequenceÚunicode_normalizer_from_stré   )ÚBaseTokenizerc                   ó„  ^ • \ rS rSrSr         SS\\\\\\	4   4      S\\\\
\\\4      4      S\S\S\\   S	\\   S
\\   S\\   S\4U 4S jjjr\S\S\4S j5       rSSS/ 4S\\\
\   4   S\	S\	S\S\
\\\4      4
S jjrSSS/ S4S\\\   \\\      4   S\	S\	S\S\
\\\4      S\\	   4S jjrSrU =r$ )ÚByteLevelBPETokenizeré
   zbByteLevelBPETokenizer

Represents a Byte-level BPE as introduced by OpenAI with their GPT-2 model
NÚvocabÚmergesÚadd_prefix_spaceÚ	lowercaseÚdropoutÚunicode_normalizerÚcontinuing_subword_prefixÚend_of_word_suffixÚtrim_offsetsc
           
      ó"  >• Ub,  Ub)  [        [        UUUU=(       d    SU=(       d    SS95      n
O[        [        5       5      n
/ nU(       a  U[        U5      /-  nU(       a  U[        5       /-  n[	        U5      S:”  a*  [	        U5      S:”  a  [        U5      U
l        O
US   U
l        [        R                  " US9U
l	        [        R                  " 5       U
l        [        R                  " U	S9U
l        SUUUUUUU	S.n[        TU ]=  X¬5        g )	NÚ )r   r   r   r   r   )r   )r    ÚByteLevelBPE)Úmodelr   r   r   r   r   r   r    )r
   r   r   r   Úlenr   Ú
normalizerr   Ú	ByteLevelÚpre_tokenizerr   Údecoderr   Úpost_processorÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   r    Ú	tokenizerÚnormalizersÚ
parametersÚ	__class__s                €Úf/home/mande/repo/quber/.venv/lib/python3.13/site-packages/tokenizers/implementations/byte_level_bpe.pyr,   ÚByteLevelBPETokenizer.__init__   s  ø€ ð Ñ Ñ!3Ü!ÜØØØ#Ø.G×.MÈ2Ø'9×'?¸Rñó‰Iô "¤#£%Ó(ˆIð ˆæØÔ7Ð8JÓKÐLÑLˆKæØœI›K˜=Ñ(ˆKô ˆ{Ó˜aÓÜ�;Ó !Ó#Ü'/°Ó'<�	Õ$à'2°1¡~�	Ô$ä"0×":Ò":ÐL\Ñ"]ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ#-×#7Ò#7À\Ñ#Rˆ	Ô ð $Ø 0Ø"ØØ"4Ø)BØ"4Ø(ñ	
ˆ
ô 	‰Ñ˜Õ/ó    Úvocab_filenameÚmerges_filenamec                 óJ   • [         R                  " X5      u  p4[        X440 UD6$ )N)r   Ú	read_filer   )r5   r6   Úkwargsr   r   s        r2   Ú	from_fileÚByteLevelBPETokenizer.from_fileJ   s"   € äŸš nÓF‰ˆÜ$ UÑ=°fÑ=Ð=r4   i0u  é   TÚfilesÚ
vocab_sizeÚmin_frequencyÚshow_progressÚspecial_tokensc                 óÎ   • [         R                  " UUUU[        R                  R	                  5       S9n[        U[        5      (       a  U/nU R                  R                  XS9  g)z%Train the model using the given files©r>   r?   r@   rA   Úinitial_alphabet)ÚtrainerN)	r   Ú
BpeTrainerr   r'   ÚalphabetÚ
isinstanceÚstrÚ
_tokenizerÚtrain)r-   r=   r>   r?   r@   rA   rE   s          r2   rK   ÚByteLevelBPETokenizer.trainO   s]   € ô ×%Ò%Ø!Ø'Ø'Ø)Ü+×5Ñ5×>Ñ>Ó@ñ
ˆô �eœS×!Ñ!Ø�GˆEØ�‰×Ñ˜eÐÒ5r4   ÚiteratorÚlengthc                 ó¢   • [         R                  " UUUU[        R                  R	                  5       S9nU R
                  R                  UUUS9  g)z(Train the model using the given iteratorrC   )rE   rN   N)r   rF   r   r'   rG   rJ   Útrain_from_iterator)r-   rM   r>   r?   r@   rA   rN   rE   s           r2   rP   Ú)ByteLevelBPETokenizer.train_from_iteratord   sT   € ô ×%Ò%Ø!Ø'Ø'Ø)Ü+×5Ñ5×>Ñ>Ó@ñ
ˆð 	�‰×+Ñ+ØØØð 	,ò 	
r4   © )	NNFFNNNNF)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r   rI   r   Úintr   r   ÚboolÚfloatr,   Ústaticmethodr:   r	   rK   r   rP   Ú__static_attributes__Ú__classcell__)r1   s   @r2   r   r   
   sÉ  ø† ñð 7;Ø>BØ!&ØØ#'Ø,0Ø37Ø,0Ø"ñ80à˜˜c 4¨¨S¨¡>Ð1Ñ2Ñ3ð80ð ˜˜s D¨¨s°C¨x©Ñ$9Ð9Ñ:Ñ;ð80ð ð	80ð
 ð80ð ˜%‘ð80ð % S™Mð80ð $,¨C¡=ð80ð % S™Mð80ð ÷80ð 80ðt ð> #ð >¸ó >ó ð>ð  ØØ"Ø79ñ6à�S˜$˜s™)�^Ñ$ð6ð ð6ð ð	6ð
 ð6ð ˜U 3¨
 ?Ñ3Ñ4õ6ð0  ØØ"Ø79Ø $ñ
à˜ ™ x°¸±Ñ'>Ð>Ñ?ð
ð ð
ð ð	
ð
 ð
ð ˜U 3¨
 ?Ñ3Ñ4ð
ð ˜‘÷
ó 
r4   r   N)Útypingr   r   r   r   r   r   Ú
tokenizersr	   r
   r   r   r   r   Útokenizers.modelsr   Útokenizers.normalizersr   r   r   Úbase_tokenizerr   r   rR   r4   r2   Ú<module>rc      s+   ðß ?× ?ç \× \Ý !ß SÑ Så )ôp
˜Mõ p
r4   