ó
    …~iI  ã                   ó†   • S SK JrJrJrJrJrJr  SSKJrJ	r	J
r
JrJr  SSKJr  SSKJrJrJrJr  SSKJr   " S S	\5      rg
)é    )ÚDictÚIteratorÚListÚOptionalÚTupleÚUnioné   )Ú
AddedTokenÚ	TokenizerÚdecodersÚpre_tokenizersÚtrainers)ÚBPE)ÚBertNormalizerÚ	LowercaseÚSequenceÚunicode_normalizer_from_stré   )ÚBaseTokenizerc                   óÂ  ^ • \ rS rSrSr         S#S\\\\\\	4   4      S\\\\
\\\4      4      S	\\\4   S
\S\\   S\S\\   S\S\4U 4S jjjr\S\S\4S j5       rSSS/S/ SS4S\\\
\   4   S\	S\	S\
\\\4      S\	S\
\   S
\\   S\4S jjrSSS/S/ SSS4S\\\   \\\      4   S\	S\	S\
\\\4      S\	S\
\   S
\\   S\S \\	   4S! jjrS"rU =r$ )$ÚCharBPETokenizeré	   a´  Original BPE Tokenizer

Represents the BPE algorithm, as introduced by Rico Sennrich
(https://arxiv.org/abs/1508.07909)

The defaults settings corresponds to OpenAI GPT BPE tokenizers and differs from the original
Sennrich subword-nmt implementation by the following options that you can deactivate:
    - adding a normalizer to clean up the text (deactivate with `bert_normalizer=False`) by:
        * removing any control characters and replacing all whitespaces by the classic one.
        * handle chinese chars by putting spaces around them.
        * strip all accents.
    - spitting on punctuation in addition to whitespaces (deactivate it with
      `split_on_whitespace_only=True`)
Nú<unk>ú</w>TÚvocabÚmergesÚ	unk_tokenÚsuffixÚdropoutÚ	lowercaseÚunicode_normalizerÚbert_normalizerÚsplit_on_whitespace_onlyc
           
      óÈ  >• Ub#  Ub   [        [        UUU[        U5      US95      n
O[        [        [        U5      XTS95      n
U
R                  [        U5      5      b  U
R	                  [        U5      /5        / nU(       a  U[        U5      /-  nU(       a  U[        SS9/-  nU(       a  U[        5       /-  n[        U5      S:”  a*  [        U5      S:”  a  [        U5      U
l
        O
US   U
l
        U	(       a  [        R                  " 5       U
l        O[        R                  " 5       U
l        [        R                   " US9U
l        SUUUUUUU	S	.n[$        TU ]M  X¬5        g )
N)r   r   Úend_of_word_suffix)r   r   r%   F)r    r   r   )r   r   )Úmodelr   r   r   r    r!   r"   r#   )r   r   ÚstrÚtoken_to_idÚadd_special_tokensr   r   r   Úlenr   Ú
normalizerr   ÚWhitespaceSplitÚpre_tokenizerÚBertPreTokenizerr   Ú
BPEDecoderÚdecoderÚsuperÚ__init__)Úselfr   r   r   r   r   r    r!   r"   r#   Ú	tokenizerÚnormalizersÚ
parametersÚ	__class__s                €Úf/home/mande/repo/quber/.venv/lib/python3.13/site-packages/tokenizers/implementations/char_level_bpe.pyr2   ÚCharBPETokenizer.__init__   sW  ø€ ð Ñ Ñ!3Ü!ÜØØØ#Ü! )›nØ'-ñó‰Iô "¤#´°I³ÈÑ"kÓlˆIà× Ñ ¤ Y£Ó0Ñ<Ø×(Ñ(¬#¨i«.Ð)9Ô:ð ˆæØÔ7Ð8JÓKÐLÑLˆKæØœN°UÑ;Ð<Ñ<ˆKæØœI›K˜=Ñ(ˆKô ˆ{Ó˜aÓÜ�;Ó !Ó#Ü'/°Ó'<�	Õ$à'2°1¡~�	Ô$æ#Ü&4×&DÒ&DÓ&FˆIÕ#ä&4×&EÒ&EÓ&GˆIÔ#ä$×/Ò/°vÑ>ˆ	Ôð Ø"ØØØ"Ø"4Ø.Ø(@ñ	
ˆ
ô 	‰Ñ˜Õ/ó    Úvocab_filenameÚmerges_filenamec                 óJ   • [         R                  " X5      u  p4[        X440 UD6$ )N)r   Ú	read_filer   )r;   r<   Úkwargsr   r   s        r8   Ú	from_fileÚCharBPETokenizer.from_file\   s"   € äŸš nÓF‰ˆÜ Ñ8°Ñ8Ð8r:   i0u  r	   iè  ÚfilesÚ
vocab_sizeÚmin_frequencyÚspecial_tokensÚlimit_alphabetÚinitial_alphabetÚshow_progressc	           
      óš   • [         R                  " UUUUUUUS9n	[        U[        5      (       a  U/nU R                  R                  XS9  g)z%Train the model using the given files©rC   rD   rE   rF   rG   r%   rH   )ÚtrainerN)r   Ú
BpeTrainerÚ
isinstancer'   Ú
_tokenizerÚtrain)
r3   rB   rC   rD   rE   rF   rG   r   rH   rK   s
             r8   rO   ÚCharBPETokenizer.traina   sT   € ô ×%Ò%Ø!Ø'Ø)Ø)Ø-Ø%Ø'ñ
ˆô �eœS×!Ñ!Ø�GˆEØ�‰×Ñ˜eÐÒ5r:   ÚiteratorÚlengthc
           
      ón   • [         R                  " UUUUUUUS9n
U R                  R                  UU
U	S9  g)z(Train the model using the given iteratorrJ   )rK   rR   N)r   rL   rN   Útrain_from_iterator)r3   rQ   rC   rD   rE   rF   rG   r   rH   rR   rK   s              r8   rT   Ú$CharBPETokenizer.train_from_iterator{   sK   € ô ×%Ò%Ø!Ø'Ø)Ø)Ø-Ø%Ø'ñ
ˆð 	�‰×+Ñ+ØØØð 	,ò 	
r:   © )	NNr   r   NFNTF)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r   r'   r   Úintr   r   r
   ÚfloatÚboolr2   Ústaticmethodr@   rO   r   rT   Ú__static_attributes__Ú__classcell__)r7   s   @r8   r   r   	   s7  ø† ñð" 7;Ø>BØ,3ØØ#'ØØ,0Ø $Ø).ñA0à˜˜c 4¨¨S¨¡>Ð1Ñ2Ñ3ðA0ð ˜˜s D¨¨s°C¨x©Ñ$9Ð9Ñ:Ñ;ðA0ð ˜˜j˜Ñ)ð	A0ð
 ðA0ð ˜%‘ðA0ð ðA0ð % S™MðA0ð ðA0ð #'÷A0ð A0ðF ð9 #ð 9¸ó 9ó ð9ð  ØØ8?°yØ"Ø&(Ø &Ø"ñ6à�S˜$˜s™)�^Ñ$ð6ð ð6ð ð	6ð
 ˜U 3¨
 ?Ñ3Ñ4ð6ð ð6ð ˜s™)ð6ð ˜‘ð6ð õ6ð:  ØØ8?°yØ"Ø&(Ø &Ø"Ø $ñ
à˜ ™ x°¸±Ñ'>Ð>Ñ?ð
ð ð
ð ð	
ð
 ˜U 3¨
 ?Ñ3Ñ4ð
ð ð
ð ˜s™)ð
ð ˜‘ð
ð ð
ð ˜‘÷
ó 
r:   r   N)Útypingr   r   r   r   r   r   Ú r
   r   r   r   r   Úmodelsr   r5   r   r   r   r   Úbase_tokenizerr   r   rV   r:   r8   Ú<module>rf      s+   ðß ?× ?ç HÕ HÝ ß ZÓ ZÝ )ôM
�}õ M
r:   