ó
    qyüiû  ã                   ó¤   • S r SSKJrJrJrJrJrJr  SSKJ	r	  SSK
Jr  SSKJr  \R                  " \5      rSSS	S
.rSS0rSr " S S\5      rS/rg)zTokenization classes for Qwen2.é    )Ú
AddedTokenÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizers)ÚBPEé   )ÚTokenizersBackend)Úloggingz
vocab.jsonz
merges.txtztokenizer.json)Ú
vocab_fileÚmerges_fileÚtokenizer_filezqwen/qwen-tokenizeri €  zn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+c                   óŠ   ^ • \ rS rSr\rSS/r\r       SS\	\
\	\4   -  S-  S\	\\	   -  S-  S\	S\	S	\	4
U 4S
 jjjrSrU =r$ )ÚQwen2Tokenizeré$   Ú	input_idsÚattention_maskNÚvocabÚmergesÚ	unk_tokenÚ	eos_tokenÚ	pad_tokenc                 ó¶  >• Ub  UOSU l         Ub  UOSS0U l        U=(       d    / U l        [        [	        U R                  U R                  S S SSSSS95      U l        [        R                  " 5       U R
                  l        [        R                  " 5       U R
                  l        [        R                  " [        R                  " [        [         5      SSS9[        R                  " U R                   SS9/5      U R
                  l        [$        T
U ]L  " SUUUUUS	.UD6  U R)                  U R*                   V	s/ s H  n	[-        U	S
S9PM     sn	5        g s  sn	f )NFú<|endoftext|>r   Ú )r   r   Údropoutr   Úcontinuing_subword_prefixÚend_of_word_suffixÚfuse_unkÚbyte_fallbackÚisolated)ÚbehaviorÚinvert)Úadd_prefix_spaceÚ	use_regex)r   Ú	bos_tokenr   r   r%   T)Úspecial© )r%   Ú_vocabÚ_mergesr   r	   Ú
_tokenizerr   Ú	ByteLevelÚdecoderr   ÚNFCÚ
normalizerr   ÚSequenceÚSplitr   ÚPRETOKENIZE_REGEXÚpre_tokenizerÚsuperÚ__init__Ú
add_tokensÚall_special_tokensr   )Úselfr   r   r   r'   r   r   r%   ÚkwargsÚtokenÚ	__class__s             €Úi/home/mande/repo/quber/.venv/lib/python3.13/site-packages/transformers/models/qwen2/tokenization_qwen2.pyr6   ÚQwen2Tokenizer.__init__)   sS  ø€ ð 5EÑ4PÑ 0ÐV[ˆÔð Ñ ñ ð   ðð 	Œð —| ˆŒÜ#ÜØ—k‘kØ—|‘|ØØØ*,Ø#%ØØ#ñ	ó
ˆŒô #+×"4Ò"4Ó"6ˆ�‰ÔÜ%0§_¢_Ó%6ˆ�‰Ô"Ü(6×(?Ò(?ä×$Ò$ÜÔ+Ó,Ø'Ø ñô
 ×(Ò(Ø%)×%:Ñ%:Ø#ñð
ó)
ˆ�‰Ô%ô 	‰Òð 	
ØØØØØ-ñ	
ð ò	
ð 	�‰Àd×F]ÒF]Ó^ÒF]¸Uœ E°4Ô8ÑF]Ñ^Õ_ùÒ^s   Ä:E)r+   r,   r*   r%   )NNr   Nr   r   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr	   ÚmodelÚstrÚdictÚintÚlistr6   Ú__static_attributes__Ú__classcell__)r<   s   @r=   r   r   $   s‘   ø† Ø)ÐØ$Ð&6Ð7ÐØ€Eð .2Ø)-Ø(ØØ(Ø(Øñ9`à�T˜#˜s˜(‘^Ñ# dÑ*ð9`ð �d˜3‘i‘ $Ñ&ð9`ð ð	9`ð ð9`ð ÷9`ö 9`ó    r   N)Ú__doc__Ú
tokenizersr   r   r   r   r   r   Útokenizers.modelsr	   Útokenization_utils_tokenizersr   Úutilsr   Ú
get_loggerr?   ÚloggerrC   ÚMAX_MODEL_INPUT_SIZESr3   r   Ú__all__r)   rM   r=   Ú<module>rW      sn   ðñ &ç Z× ZÝ !å >Ý ð 
×	Ò	˜HÓ	%€ð ØØ&ñÐ ð /°Ð6Ð ð JÐ ô>`Ð&ô >`ðB Ð
�rM   