ó
    …~i‰  ã                   óz   • S SK JrJrJrJrJrJr  S SKJrJ	r	J
r
JrJr  S SKJr  S SKJr  SSKJr   " S S\5      rg	)
é    )ÚDictÚIteratorÚListÚOptionalÚTupleÚUnion)Ú
AddedTokenÚ	TokenizerÚdecodersÚpre_tokenizersÚtrainers)ÚBPE)ÚNFKCé   )ÚBaseTokenizerc                   óž  ^ • \ rS rSrSr       S S\\\\\\	4   4      S\\\\
\\\4      4      S\\\4   S	\S
\S\\   S\\   4U 4S jjjr\S\S\4S j5       rSSS/S/ S4S\\\
\   4   S\	S\	S\
\\\4      S\	S\
\   S\4S jjrSSS/S/ SS4S\\\   \\\      4   S\	S\	S\
\\\4      S\	S\
\   S\S\\	   4S jjrSrU =r$ )!ÚSentencePieceBPETokenizeré
   zjSentencePiece BPE Tokenizer

Represents the BPE algorithm, with the pretokenization used by SentencePiece
Nú<unk>TÚvocabÚmergesÚ	unk_tokenÚreplacementÚadd_prefix_spaceÚdropoutÚfuse_unkc           
      ó”  >• Ub  Ub  [        [        XXcUS95      nO[        [        XcUS95      nUR                  [        U5      5      b  UR	                  [        U5      /5        [        5       Ul        U(       a  SOSn	[        R                  " XIS9Ul	        [        R                  " XIS9Ul        SUUUUS.n
[        TU ]5  XŠ5        g )N)r   r   r   ÚalwaysÚnever)r   Úprepend_schemeÚSentencePieceBPE)Úmodelr   r   r   r   )r
   r   Útoken_to_idÚstrÚadd_special_tokensr   Ú
normalizerr   Ú	MetaspaceÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   Ú	tokenizerr    Ú
parametersÚ	__class__s              €Úi/home/mande/repo/quber/.venv/lib/python3.13/site-packages/tokenizers/implementations/sentencepiece_bpe.pyr+   Ú"SentencePieceBPETokenizer.__init__   sÀ   ø€ ð Ñ Ñ!3Ü!¤# e¸WÐdlÑ"mÓn‰Iä!¤#¨gÐU]Ñ"^Ó_ˆIà× Ñ ¤ Y£Ó0Ñ<Ø×(Ñ(¬#¨i«.Ð)9Ô:ä#›vˆ	ÔÞ%5™¸7ˆÜ"0×":Ò":À{Ñ"rˆ	ÔÜ$×.Ò.¸;Ñfˆ	Ôð (Ø"Ø&Ø 0Øñ
ˆ
ô 	‰Ñ˜Õ/ó    Úvocab_filenameÚmerges_filenamec                 óJ   • [         R                  " X5      u  p4[        X440 UD6$ )N)r   Ú	read_filer   )r3   r4   Úkwargsr   r   s        r0   Ú	from_fileÚ#SentencePieceBPETokenizer.from_file1   s"   € äŸš nÓF‰ˆÜ(¨ÑA¸&ÑAÐAr2   i0u  é   iè  ÚfilesÚ
vocab_sizeÚmin_frequencyÚspecial_tokensÚlimit_alphabetÚinitial_alphabetÚshow_progressc           	      ó˜   • [         R                  " UUUUUUS9n[        U[        5      (       a  U/nU R                  R                  XS9  g)z%Train the model using the given files©r<   r=   r>   r?   r@   rA   )ÚtrainerN)r   Ú
BpeTrainerÚ
isinstancer$   Ú
_tokenizerÚtrain)	r,   r;   r<   r=   r>   r?   r@   rA   rD   s	            r0   rH   ÚSentencePieceBPETokenizer.train6   sQ   € ô ×%Ò%Ø!Ø'Ø)Ø)Ø-Ø'ñ
ˆô �eœS×!Ñ!Ø�GˆEØ�‰×Ñ˜eÐÒ5r2   ÚiteratorÚlengthc	           	      ól   • [         R                  " UUUUUUS9n	U R                  R                  UU	US9  g)z(Train the model using the given iteratorrC   )rD   rK   N)r   rE   rG   Útrain_from_iterator)
r,   rJ   r<   r=   r>   r?   r@   rA   rK   rD   s
             r0   rM   Ú-SentencePieceBPETokenizer.train_from_iteratorN   sH   € ô ×%Ò%Ø!Ø'Ø)Ø)Ø-Ø'ñ
ˆð 	�‰×+Ñ+ØØØð 	,ò 	
r2   © )NNr   u   â–�TNF)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r   r$   r   Úintr   r   r	   ÚboolÚfloatr+   Ústaticmethodr8   rH   r   rM   Ú__static_attributes__Ú__classcell__)r/   s   @r0   r   r   
   sô  ø† ñð 7;Ø>BØ,3Ø Ø!%Ø#'Ø#(ñ0à˜˜c 4¨¨S¨¡>Ð1Ñ2Ñ3ð0ð ˜˜s D¨¨s°C¨x©Ñ$9Ð9Ñ:Ñ;ð0ð ˜˜j˜Ñ)ð	0ð
 ð0ð ð0ð ˜%‘ð0ð ˜4‘.÷0ð 0ðB ðB #ð B¸ó Bó ðBð  ØØ8?°yØ"Ø&(Ø"ñ6à�S˜$˜s™)�^Ñ$ð6ð ð6ð ð	6ð
 ˜U 3¨
 ?Ñ3Ñ4ð6ð ð6ð ˜s™)ð6ð õ6ð6  ØØ8?°yØ"Ø&(Ø"Ø $ñ
à˜ ™ x°¸±Ñ'>Ð>Ñ?ð
ð ð
ð ð	
ð
 ˜U 3¨
 ?Ñ3Ñ4ð
ð ð
ð ˜s™)ð
ð ð
ð ˜‘÷
ó 
r2   r   N)Útypingr   r   r   r   r   r   Ú
tokenizersr	   r
   r   r   r   Útokenizers.modelsr   Útokenizers.normalizersr   Úbase_tokenizerr   r   rO   r2   r0   Ú<module>r`      s(   ðß ?× ?ç PÕ PÝ !Ý 'å )ô]
 õ ]
r2   