ó
    pyüib1 ã                   ó\  • S r SSKrSSKJr  SSKJr  SSKJr  SSKJ	r	J
r
JrJrJrJrJr  SSKJrJrJr  SSKJr  S	S
KJrJrJrJr  S	SKJr  \R8                  " \5      r/ SQr\/ SQ-   r S¦S jr!S\"S\#4S jr$S§S\\#   S-  4S jjr% " S S5      r& " S S\&5      r'S\#S\"4S jr( " S S5      r) " S S\)5      r* " S S\)5      r+ " S  S!\)5      r, " S" S#\)5      r- " S$ S%\)5      r. " S& S'\)5      r/ " S( S)\)5      r0 " S* S+\)5      r1 " S, S-\)5      r2 " S. S/\)5      r3 " S0 S1\)5      r4 " S2 S3\)5      r5 " S4 S5\55      r6 " S6 S7\55      r7 " S8 S9\55      r8 " S: S;\55      r9 " S< S=\55      r: " S> S?\55      r; " S@ SA\55      r< " SB SC\55      r= " SD SE\55      r> " SF SG\55      r? " SH SI\55      r@ " SJ SK\55      rA " SL SM\55      rB " SN SO\55      rC " SP SQ\55      rD " SR SS\55      rE " ST SU\)5      rF " SV SW\55      rG " SX SY\)5      rH " SZ S[\)5      rI " S\ S]\)5      rJ " S^ S_\55      rK " S` Sa\55      rL " Sb Sc\55      rM " Sd Se\)5      rN " Sf Sg\55      rO " Sh Si\55      rP " Sj Sk\55      rQSl rR " Sm Sn5      rS " So Sp5      rT0 Sq\6_Sr\2_Ss\7_St\*_Su\G_Sv\J_Sw\8_Sx\H_Sy\/_Sz\*_S{\4_S|\9_S}\*_S~\*_S\*_S€\*_S�\*_0 S‚\6_Sƒ\,_S„\/_S…\0_S†\*_S‡\*_Sˆ\2_S‰\>_SŠ\2_S‹\2_SŒ\*_S�\N_SŽ\:_S�\;_S�\-_S‘\*_S’\2_E0 S“\<_S”\._S•\C_S–\1_S—\@_S˜\A_S™\2_Sš\3_S›\=_Sœ\*_S�\D_Sž\E_SŸ\F_S \>_S¡\?_S¢\+_S£\K_E\M\M\L\MS¤.ErUS¨S\4S¥ jjrVg)©zù
Utilities to convert slow tokenizers in their fast tokenizers counterparts.

All the conversions are grouped here to gather SentencePiece dependencies outside of the fast tokenizers files and
allow to make our dependency on SentencePiece optional.
é    N)Ú
Collection)Ú	lru_cache)Úversion)Ú
AddedTokenÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramÚ	WordPiece)Útqdmé   )Úis_protobuf_availableÚis_sentencepiece_availableÚloggingÚrequires_backends)ÚPROTOBUF_IMPORT_ERROR)Úar_ARÚcs_CZÚde_DEÚen_XXÚes_XXÚet_EEÚfi_FIÚfr_XXÚgu_INÚhi_INÚit_ITÚja_XXÚkk_KZÚko_KRÚlt_LTÚlv_LVÚmy_MMÚne_NPÚnl_XXÚro_ROÚru_RUÚsi_LKÚtr_TRÚvi_VNÚzh_CN)Úaf_ZAÚaz_AZÚbn_INÚfa_IRÚhe_ILÚhr_HRÚid_IDÚka_GEÚkm_KHÚmk_MKÚml_INÚmn_MNÚmr_INÚpl_PLÚps_AFÚpt_XXÚsv_SEÚsw_KEÚta_INÚte_INÚth_THÚtl_XXÚuk_UAÚur_PKÚxh_ZAÚgl_ESÚsl_SIc                 ó8  • [        5       (       a  SSKJn  U$ [        5       (       aV  SS Kn[
        R                  " UR                  R                  5      [
        R                  " S5      :  a  SSK	Jn  U$ SSK	J
n  U$ [        [        R                  " U 5      5      e)Nr   )Úsentencepiece_model_pb2z4.0.0)Úsentencepiece_model_pb2_new)r   ÚsentencepiecerL   r   Úgoogle.protobufr   ÚparseÚprotobufÚ__version__Útransformers.utilsrM   ÚImportErrorr   Úformat)Úerror_messagerL   Úgoogles      Ú`/home/mande/repo/quber/.venv/lib/python3.13/site-packages/transformers/convert_slow_tokenizer.pyÚimport_protobufrY   _   sr   € Ü!×#Ñ#Ý9à&Ð&Ü×ÑÛä�=Š=˜Ÿ™×4Ñ4Ó5¼¿ºÀgÓ8NÓNÝBð 'Ð&õ bØ&Ð&äÔ/×6Ò6°}ÓEÓFÐFó    Úadd_prefix_spaceÚreturnc                 óH   • U (       a  Sn[        USS5      (       d  SnU$ SnU$ )NÚalwaysÚlegacyTÚfirstÚnever)Úgetattr)r[   Úoriginal_tokenizerÚprepend_schemes      rX   Ú_get_prepend_schemere   p   s4   € ÞØ!ˆÜÐ)¨8°T×:Ñ:Ø$ˆNð Ðð !ˆØÐrZ   Úskip_tokensc                 óè  ^ • Ub  [        U5      O	[        5       nUS LnU(       a  [        U5      OT n/ nUR                  5        H€  u  pVXR;   a  M  / n[        S[	        U5      5       H:  nUS U XXS  p©X’;   d  X¢;   a  M  U	T ;   d  M  U
T ;   d  M'  UR                  XšU45        M<     [        UU 4S jS9nUR                  U5        M‚     [        US US9nU Vs/ s H  o»S   US   4PM     nnU$ s  snf )Nr   c                 ó$   >• TU S      TU S      4$ ©Nr   r   © )ÚxÚvocabs    €rX   Ú<lambda>Ú!generate_merges.<locals>.<lambda>Š   s   ø€ ¨U°1°Q±4©[¸%ÀÀ!Á¹+Ñ,FrZ   ©Úkeyc                 óB   • U S   [        U S   5      [        U S   5      4$ )Né   r   r   )Úlen©Úvals    rX   rm   rn   �   s   € ¨S°©V´S¸¸Q¹³[Ä#ÀcÈ!ÁfÃ+Ñ,NrZ   ©rp   Úreverser   )ÚsetÚdictÚitemsÚrangers   ÚappendÚsortedÚextend)rl   Úvocab_scoresrf   rw   ÚmergesÚmergeÚpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rru   s   `           rX   Úgenerate_mergesr‡   z   s  ø€ Ø&1Ñ&=”#�kÔ"Ä3Ã5€KØ $Ð&€GÞ)0”4˜Ô%°e€Là€FØ*×0Ñ0Ö2ÑˆØÓÙØˆÜ˜1œc %›jÖ)ˆEØ$ V e˜}¨e°F¨m�WØÓ%¨Ó)?ÙØ˜%Õ G¨uÕ$4Ø—‘˜g°Ð<Ö=ñ *ô �uÔ"FÑGˆØ�‰�eÖñ 3ô �FÑ NÐX_Ñ`€FÙ*0Ó1ª& 3�1‰v�s˜1‘vÓ©&€FÐ1Ø€Mùò 2s   ÃC/c                   óN   • \ rS rSrSrS\4S jrS\\\\	4   \
\   4   4S jrSrg)	ÚSentencePieceExtractoré’   zd
Extractor implementation for SentencePiece trained models. https://github.com/google/sentencepiece
Úmodelc                 óþ   • [        U S5        [        U S5        [        5       nUR                  5       n[        US5       nUR	                  UR                  5       5        S S S 5        X0l        g ! , (       d  f       N= f)NrN   rQ   Úrb)r   rY   Ú
ModelProtoÚopenÚParseFromStringÚreadÚproto)Úselfr‹   Ú	model_pb2ÚmÚfs        rX   Ú__init__ÚSentencePieceExtractor.__init__—   sa   € Ü˜$ Ô0Ü˜$ 
Ô+ô $Ó%ˆ	à× Ñ Ó"ˆÜ�%˜Ô !Ø×Ñ˜aŸf™f›hÔ'÷ à�
÷ Õús   ¿ A.Á.
A<r\   c                 óš  • U R                   R                  R                    Uc0  SSKJnJn  U R                   R                  R                  S:X  a  UOUnU R                   R                   Vs/ s H  oUR                  UR                  4PM     nnUR                  S:w  a(  U R                   R                  R                  US'   XbS'   O:SSKJn  [        U5       VV	V
s0 s H
  u  nu  pšX˜_M     nn	nn
U" U5      nXbS'   X²S	'   [        U R                   R                  5       VVs/ s H2  u  pÍUR                  S
;   d  M  XÍR                  UR                  S:H  4PM4     nnn[        US S9 VVVs/ s H  u  pÏn[!        USUS9PM     snnnUS'   [#        U R                   R$                  SS5      US'   U$ s  snf s  sn
n	nf s  snnf s  snnnf )úª
By default will return vocab and merges with respect to their order, by sending `vocab_scores` we're going to
order the merges with respect to the piece scores instead.
Nr   )r   r   r   r   Úunk_idrl   )r‡   r€   ©é   é   r�   c                 ó   • U S   $ ©Nr   rj   ©rk   s    rX   rm   Ú0SentencePieceExtractor.extract.<locals>.<lambda>Á   s   € ÈQÈqÊTrZ   ro   F©Ú
normalizedÚspecialÚadditional_special_tokensÚprecompiled_charsmapÚ_spm_precompiled_charsmap)r’   Útrainer_specr›   Útokenizers.modelsr   r   Ú
model_typeÚpiecesÚpieceÚscoreÚ__name__Útokenization_utils_baser‡   Ú	enumerateÚtyper}   r   rb   Únormalizer_spec)r“   r«   Úkwargsr   r   r­   rl   r‡   ÚiÚwordr®   r€   ÚidÚpÚspm_added_tokensÚtokenr¥   s                    rX   ÚextractÚSentencePieceExtractor.extract£   s¡  € ð
 	�
‰
×Ñ×&Ò&ØÑß6à$(§J¡J×$;Ñ$;×$FÑ$FÈ!Ó$K™ÐQTˆJØ9=¿¹×9JÒ9JÓKÒ9J°—+‘+˜uŸ{™{Ó+Ñ9JˆÐKà×Ñ %Ó'Ø#Ÿz™z×6Ñ6×=Ñ=ˆF�8ÑØ#�7ŠOå@ä5>¸uÔ5EÕFÒ5EÑ!1 ¡M T�T’WÑ5EˆEÒFÙ$ UÓ+ˆFØ#�7‰OØ%�8Ñô ENÈdÏjÉj×N_ÑN_ÔD`ÔuÒD`¹5¸2Ðde×djÑdjÐntÑdtÓ6˜R§¡¨!¯&©&°A©+Ó6ÑD`ÐÑuô '-Ð-=Á>Ò&Rõ/
â&RÑ"�˜7ô �u¨¸Ô@Ù&Ró/
ˆÐ*Ñ+ô /6°d·j±j×6PÑ6PÐRhÐjnÓ.oˆÐ*Ñ+Øˆùò/ Lùô Gùó vùô/
s   Á,!F4ÃF9Ä#G Ä=G Å.G©r’   N)r¯   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ústrr—   Útuplery   ÚintÚlistr»   Ú__static_attributes__rj   rZ   rX   r‰   r‰   ’   s5   † ñð
˜cô 
ð!¨u°T¸#¸s¸(±^ÀTÈ%Á[Ð5PÑ/Q÷ !rZ   r‰   c                   ó@   • \ rS rSrSS\\\\4   \\   4   4S jjr	Sr
g)ÚGemmaSentencePieceExtractoréÇ   Nr\   c                 óä   • U R                   n[        UR                  5       5       Vs0 s H  o2R                  U5      U_M     nnSU;  a  UR	                  S5      US'   [        XA5      nXE4$ s  snf )rš   Ú	ú<0x09>)Úspr{   ÚGetPieceSizeÚid_to_pieceÚgetr‡   )r“   r   rÍ   r„   rl   r€   s         rX   r»   Ú#GemmaSentencePieceExtractor.extractÈ   so   € ð
 �W‰WˆÜ;@ÀÇÁÓARÔ;SÓTÒ;S°%—‘ Ó&¨Ò-Ñ;SˆÐTð �uÓØŸ)™) HÓ-ˆE�$‰KÜ  Ó5ˆØˆ}Ðùò Us   ¨A-rj   ©N)r¯   r¾   r¿   rÀ   rÃ   ry   rÂ   rÄ   rÅ   r»   rÆ   rj   rZ   rX   rÈ   rÈ   Ç   s)   † ñ¨E°$°s¸C°x±.À$ÀuÁ+Ð2MÑ,N÷ ð rZ   rÈ   r­   c                 óz   • [        U 5      S:  =(       d'    U S   S:g  =(       d    U S   R                  5       (       + $ )Nrr   éÿÿÿÿÚ,éþÿÿÿ)rs   Úisdigit)r­   s    rX   Úcheck_number_commarØ   Ø   s3   € Üˆu‹:˜‰>×H˜U 2™Y¨#Ñ-×H°U¸2±Y×5FÑ5FÓ5HÔ1HÐHrZ   c                   ó(   • \ rS rSrS rS\4S jrSrg)Ú	ConverteréÜ   c                 ó   • Xl         g rÒ   ©rc   )r“   rc   s     rX   r—   ÚConverter.__init__Ý   s   € Ø"4ÕrZ   r\   c                 ó   • [        5       erÒ   )ÚNotImplementedError©r“   s    rX   Ú	convertedÚConverter.convertedà   s   € Ü!Ó#Ð#rZ   rÝ   N)r¯   r¾   r¿   rÀ   r—   r   râ   rÆ   rj   rZ   rX   rÚ   rÚ   Ü   s   † ò5ð$˜9÷ $rZ   rÚ   c                   ó"   • \ rS rSrS\4S jrSrg)ÚBertConverteréä   r\   c           	      ób  • U R                   R                  n[        [        U[	        U R                   R
                  5      S95      nSnSnSn[        U R                   S5      (       a`  U R                   R                  R                  nU R                   R                  R                  nU R                   R                  R                  n[        R                  " SUUUS9Ul        [        R                  " 5       Ul        [	        U R                   R"                  5      n[	        U R                   R$                  5      nU R                   R&                  nU R                   R(                  n	[*        R,                  " U SU S3U SU SU S	3Xh4Xy4/S
9Ul        [0        R                  " SS9Ul        U$ )N©Ú	unk_tokenFÚbasic_tokenizerT©Ú
clean_textÚhandle_chinese_charsÚstrip_accentsÚ	lowercaseú:0 $A:0 ú:0ú:0 $B:1 ú:1©ÚsingleÚpairÚspecial_tokensú##©Úprefix©rc   rl   r   r   rÂ   ré   Úhasattrrê   Útokenize_chinese_charsrî   Údo_lower_caser
   ÚBertNormalizerÚ
normalizerr   ÚBertPreTokenizerÚpre_tokenizerÚ	cls_tokenÚ	sep_tokenÚcls_token_idÚsep_token_idr   ÚTemplateProcessingÚpost_processorr	   Údecoder©
r“   rl   Ú	tokenizerrý   rî   rþ   ÚclsÚsepr  r  s
             rX   râ   ÚBertConverter.convertedå   ó„  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ñ`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ñ>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ñ	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5àÐ#ØÐ#ðñ$
ˆ	Ô ô %×.Ò.°dÑ;ˆ	ÔàÐrZ   rj   N©r¯   r¾   r¿   rÀ   r   râ   rÆ   rj   rZ   rX   rå   rå   ä   ó   † ð#˜9÷ #rZ   rå   c                   ó"   • \ rS rSrS\4S jrSrg)ÚSplinterConverteri  r\   c           
      óv  • U R                   R                  n[        [        U[	        U R                   R
                  5      S95      nSnSnSn[        U R                   S5      (       a`  U R                   R                  R                  nU R                   R                  R                  nU R                   R                  R                  n[        R                  " SUUUS9Ul        [        R                  " 5       Ul        [	        U R                   R"                  5      n[	        U R                   R$                  5      n[	        U R                   R&                  5      nSn	U R                   R(                  n
U R                   R*                  nU R                   R,                  nU R                   R/                  S5      nU R                   R0                  S:X  a  U SU S	U	 S	U S
U S3
nOU SU S
U S	U	 S	U S3
n[2        R4                  " U SU S3UXj4X{4XŒ4X�4/S9Ul        [8        R                  " SS9Ul        U$ )Nrè   Frê   Trë   Ú.Úrightrð   Ú rò   ró   rñ   rô   rø   rù   )rc   rl   r   r   rÂ   ré   rü   rê   rý   rî   rþ   r
   rÿ   r   r   r  r  r  r  Úquestion_tokenr  r  Úquestion_token_idÚconvert_tokens_to_idsÚpadding_sider   r  r  r	   r	  )r“   rl   r  rý   rî   rþ   r  r  ÚquestionÚdotr  r  r  Údot_token_idrö   s                  rX   râ   ÚSplinterConverter.converted  s  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ñ`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ñ>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ñ	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆÜ�t×.Ñ.×=Ñ=Ó>ˆØˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆØ ×3Ñ3×EÑEÐØ×.Ñ.×DÑDÀSÓIˆà×"Ñ"×/Ñ/°7Ó:Ø�U˜( 8 *¨A¨c¨U°!°C°5¸ÀÀÀRÐH‰Dà�U˜( 3 % x°¨z¸¸3¸%¸qÀÀÀRÐHˆDä#-×#@Ò#@Ø�U˜( 3 % rÐ*ØàÐ#ØÐ#ØÐ-ØÐ#ð	ñ	$
ˆ	Ô ô %×.Ò.°dÑ;ˆ	ÔàÐrZ   rj   Nr  rj   rZ   rX   r  r    s   † ð.˜9÷ .rZ   r  c                   ó"   • \ rS rSrS\4S jrSrg)ÚFunnelConverteri=  r\   c           	      ób  • U R                   R                  n[        [        U[	        U R                   R
                  5      S95      nSnSnSn[        U R                   S5      (       a`  U R                   R                  R                  nU R                   R                  R                  nU R                   R                  R                  n[        R                  " SUUUS9Ul        [        R                  " 5       Ul        [	        U R                   R"                  5      n[	        U R                   R$                  5      nU R                   R&                  nU R                   R(                  n	[*        R,                  " U SU S3U SU SU S	3Xh4Xy4/S
9Ul        [0        R                  " SS9Ul        U$ )Nrè   Frê   Trë   z:2 $A:0 rñ   rò   ró   rô   rø   rù   rû   r
  s
             rX   râ   ÚFunnelConverter.converted>  r  rZ   rj   Nr  rj   rZ   rX   r!  r!  =  r  rZ   r!  c                   ó"   • \ rS rSrS\4S jrSrg)ÚMPNetConverterid  r\   c                 óh  • U R                   R                  n[        [        U[	        U R                   R
                  5      S95      nSnSnSn[        U R                   S5      (       a`  U R                   R                  R                  nU R                   R                  R                  nU R                   R                  R                  n[        R                  " SUUUS9Ul        [        R                  " 5       Ul        [	        U R                   R"                  5      n[	        U R                   R$                  5      nU R                   R&                  nU R                   R(                  n	[*        R,                  " U SU S3U SU SU S	U S
3Xh4Xy4/S9Ul        [0        R                  " SS9Ul        U$ )Nrè   Frê   Trë   rð   rñ   z:0 rò   ró   rô   rø   rù   rû   r
  s
             rX   râ   ÚMPNetConverter.convertede  sŠ  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ñ`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ñ>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ñ	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   S¨¨¨X°c°U¸"Ð=àÐ#ØÐ#ðñ$
ˆ	Ô ô %×.Ò.°dÑ;ˆ	ÔàÐrZ   rj   Nr  rj   rZ   rX   r%  r%  d  r  rZ   r%  c                   ó"   • \ rS rSrS\4S jrSrg)ÚOpenAIGPTConverteri‹  r\   c                 óü  • U R                   R                  n[        U R                   R                  R	                  5       5      nU R                   R
                  n[        [        UUS [        U5      SSS95      nUR                  [        U5      5      b  UR                  [        U5      /5        [        R                  " SS9Ul        [        R                  " 5       Ul        ["        R$                  " SS9Ul        U$ )Nú</w>F)rl   r€   Údropoutré   Úend_of_word_suffixÚfuse_unkT)rï   ©Úsuffix)rc   ÚencoderrÅ   Ú	bpe_ranksÚkeysré   r   r   rÂ   Útoken_to_idÚadd_special_tokensr
   rÿ   r   r   r  r  r	   Ú
BPEDecoderr	  ©r“   rl   r€   ré   r  s        rX   râ   ÚOpenAIGPTConverter.convertedŒ  sÔ   € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØÜ˜i›.Ø#)Øñó	
ˆ	ð × Ñ ¤ Y£Ó0Ñ<Ø×(Ñ(¬#¨i«.Ð)9Ô:ä*×9Ò9ÀDÑIˆ	ÔÜ"0×"AÒ"AÓ"Cˆ	ÔÜ$×/Ò/°vÑ>ˆ	ÔàÐrZ   rj   Nr  rj   rZ   rX   r)  r)  ‹  s   † ð˜9÷ rZ   r)  c                   óT   • \ rS rSrSS\\\4   S-  S\\\\4      S-  S\	4S jjr
Srg)	ÚGPT2Converteri¦  Nrl   r€   r\   c                 óZ  • U(       d  U R                   R                  nU(       d  [        U R                   R                  5      n[	        [        UUS SSSS95      n[        U R                   SS5      n[        R                  " US9Ul	        [        R                  " 5       Ul        [        U R                   SS5      (       aQ  U R                   R                  nU R                   R                  n[        R                  " U S3U S3XV4/S	9Ul        U$ [        R                  " SS
9Ul        U$ )NÚ F©rl   r€   r,  Úcontinuing_subword_prefixr-  r.  r[   ©r[   Úadd_bos_tokenz:0 $A:0z:0 $A:0 $B:1rô   ©Útrim_offsets)rc   r1  rÅ   r2  r   r   rb   r   Ú	ByteLevelr  r	   r	  Ú	bos_tokenÚbos_token_idr   r  r  )r“   rl   r€   r  r[   ÚbosrE  s          rX   râ   ÚGPT2Converter.converted§  s  € ÞØ×+Ñ+×3Ñ3ˆEÞÜ˜$×1Ñ1×;Ñ;Ó<ˆFäÜØØØØ*,Ø#%Øñó	
ˆ	ô # 4×#:Ñ#:Ð<NÐPUÓVÐÜ"0×":Ò":ÐL\Ñ"]ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ�4×*Ñ*¨O¸U×CÑCØ×)Ñ)×3Ñ3ˆCØ×2Ñ2×?Ñ?ˆLÜ'1×'DÒ'DØ˜˜g�Ø�u˜LÐ)àÐ'ð ñ(ˆIÔ$ð Ðô (2×';Ò';ÈÑ'OˆIÔ$ØÐrZ   rj   ©NN©r¯   r¾   r¿   rÀ   ry   rÂ   rÄ   rÅ   rÃ   r   râ   rÆ   rj   rZ   rX   r:  r:  ¦  sE   † ñ"˜t C¨ H™~°Ñ4ð "ÀTÈ%ÐPSÐUXÐPXÉ/ÑEZÐ]aÑEað "Ðmv÷ "ð "rZ   r:  c                   ó"   • \ rS rSrS\4S jrSrg)ÚHerbertConverteriÌ  r\   c           
      ó~  • SnSnU R                   R                  n[        U R                   R                  R	                  5       5      nXS   S   ;   a  USS  n[        [        UUS U R                   R                  US95      n[        R                  " SSS9Ul
        [        R                  " 5       Ul        [        R                  " US9Ul        ["        R$                  " U R                   R&                  U R                   R(                  4U R                   R*                  U R                   R,                  4S	9Ul        U$ )
Nz	#version:r+  r   r   )r,  ré   r-  F)rï   rî   r/  )r  r  )rc   r1  rÅ   r2  r3  r   r   ré   r
   rÿ   r   r   r  r  r	   r6  r	  r   ÚBertProcessingr  r  r  r  r  )r“   Útokenizer_info_strÚtoken_suffixrl   r€   r  s         rX   râ   ÚHerbertConverter.convertedÍ  s  € Ø(ÐØˆà×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ¨¡¨1¡Ó-Ø˜A˜B�ZˆFäÜØØØØ×1Ñ1×;Ñ;Ø#/ñó
ˆ	ô  +×9Ò9ÀEÐY^Ñ_ˆ	ÔÜ"0×"AÒ"AÓ"Cˆ	ÔÜ$×/Ò/°|ÑDˆ	ÔÜ#-×#<Ò#<Ø×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYñ$
ˆ	Ô ð
 ÐrZ   rj   Nr  rj   rZ   rX   rK  rK  Ì  ó   † ð˜9÷ rZ   rK  c                   óT   • \ rS rSrSS\\\4   S-  S\\\\4      S-  S\	4S jjr
Srg)	ÚQwen2Converterië  Nrl   r€   r\   c                 ó8  • U(       d  U R                   R                  nU(       d-  [        U R                   R                  R	                  5       5      n[        [        UUS S SSSSS95      n[        R                  " 5       Ul	        [        R                  " [        R                  " [        S5      SSS9[        R                  " [        U R                   SS5      SS9/5      Ul        ["        R                  " 5       Ul        [&        R                  " SS	9Ul        U$ )
Nr<  F)rl   r€   r,  ré   r>  r-  r.  Úbyte_fallbackzn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+Úisolated©ÚbehaviorÚinvertr[   ©r[   Ú	use_regexrA  )rc   r1  rÅ   r2  r3  r   r   r
   ÚNFCr   r   ÚSequenceÚSplitr   rC  rb   r  r	   r	  r   r  )r“   rl   r€   r  s       rX   râ   ÚQwen2Converter.convertedì  s  € ÞØ×+Ñ+×3Ñ3ˆEÞÜ˜$×1Ñ1×;Ñ;×@Ñ@ÓBÓCˆFäÜØØØØØ*,Ø#%ØØ#ñ	ó
ˆ	ô  +ŸšÓ0ˆ	Ôä"0×"9Ò"9ä×$Ò$Üð Nóð (Ø ñô ×(Ò(Ü%,¨T×-DÑ-DÐFXÐZ_Ó%`Ø#ñðó#
ˆ	Ôô  %×.Ò.Ó0ˆ	ÔÜ#-×#7Ò#7ÀUÑ#Kˆ	Ô àÐrZ   rj   rH  rI  rj   rZ   rX   rS  rS  ë  sE   † ñ(˜t C¨ H™~°Ñ4ð (ÀTÈ%ÐPSÐUXÐPXÉ/ÑEZÐ]aÑEað (Ðmv÷ (ð (rZ   rS  c                   ó"   • \ rS rSrS\4S jrSrg)ÚRobertaConverteri  r\   c                 óÈ  • U R                   nUR                  n[        UR                  R	                  5       5      n[        [        UUS SSSS95      n[        R                  " UR                  S9Ul
        [        R                  " 5       Ul        [        R                  " UR                  UR                   4UR"                  UR$                  4UR                  SS9Ul        U$ )Nr<  Fr=  r?  T©r  r  r[   rB  )rc   r1  rÅ   r2  r3  r   r   r   rC  r[   r  r	   r	  r   ÚRobertaProcessingr  r  r  r  r  ©r“   Úotrl   r€   r  s        rX   râ   ÚRobertaConverter.converted  sÄ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øñó	
ˆ	ô #1×":Ò":ÈB×L_ÑL_Ñ"`ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ#-×#?Ò#?Ø—‘˜rŸ™Ð/Ø—‘˜rŸ™Ð/Ø×0Ñ0Øñ	$
ˆ	Ô ð ÐrZ   rj   Nr  rj   rZ   rX   ra  ra    ó   † ð˜9÷ rZ   ra  c                   ó"   • \ rS rSrS\4S jrSrg)ÚRoFormerConverteri4  r\   c           	      óJ  • SSK Jn  U R                  R                  n[	        [        U[        U R                  R                  5      S95      nSnSn[        U R                  S5      (       a@  U R                  R                  R                  nU R                  R                  R                  n[        R                  " SSUUS9Ul        [        R                   R#                  U" U5      5      Ul        [        U R                  R&                  5      n[        U R                  R(                  5      nU R                  R*                  nU R                  R,                  n	[.        R0                  " U SU S	3U SU S
U S3Xh4Xy4/S9Ul        [4        R
                  " SS9Ul        U$ )Nr   )ÚJiebaPreTokenizerrè   Frê   Trë   rð   rñ   rò   ró   rô   rø   rù   )Ú"models.roformer.tokenization_utilsrl  rc   rl   r   r   rÂ   ré   rü   rê   rî   rþ   r
   rÿ   r   r   ÚPreTokenizerÚcustomr  r  r  r  r  r   r  r  r	   r	  )
r“   rl  rl   r  rî   rþ   r  r  r  r  s
             rX   râ   ÚRoFormerConverter.converted5  sx  € ÝIà×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ñ`Óaˆ	àˆØˆÜ�4×*Ñ*Ð,=×>Ñ>Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!&Ø'Ø#ñ	 
ˆ	Ôô #1×"=Ñ"=×"DÑ"DÑEVÐW\ÓE]Ó"^ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5àÐ#ØÐ#ðñ$
ˆ	Ô ô %×.Ò.°dÑ;ˆ	ÔàÐrZ   rj   Nr  rj   rZ   rX   rj  rj  4  r  rZ   rj  c                   ó"   • \ rS rSrS\4S jrSrg)ÚDebertaConverteri[  r\   c                 óÊ  • U R                   nUR                  n[        UR                  R	                  5       5      n[        [        UUS SSSS95      n[        R                  " UR                  S9Ul
        [        R                  " 5       Ul        [        R                  " SSSU R                   R                  S5      4SU R                   R                  S5      4/S	9Ul        U$ )
Nr<  Fr=  r?  ú[CLS]:0 $A:0 [SEP]:0ú![CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1ú[CLS]ú[SEP]rô   )rc   r1  rÅ   r2  r3  r   r   r   rC  r[   r  r	   r	  r   r  r  r  re  s        rX   râ   ÚDebertaConverter.converted\  sÕ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øñó	
ˆ	ô #1×":Ò":ÈB×L_ÑL_Ñ"`ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ#-×#@Ò#@Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðñ$
ˆ	Ô ð ÐrZ   rj   Nr  rj   rZ   rX   rr  rr  [  rQ  rZ   rr  c                   ó”   ^ • \ rS rSrSr\r0 r\SS j5       r	\
SS j5       rU 4S jrS rS rS rS	 rS
 rS rS rS\4S jrSrU =r$ )ÚSpmConverteriz  Fc                 ó¦  • U R                   R                  nU R                   R                  nU R                  R                  n[        U[        5      (       a!  [        [        UU=(       d    / USUSS95      nOe[        U[        5      (       aO  U(       aH  [        US   [        [        -  5      (       a)  [        [        UU R                   R                  US95      nOg[        R                  " SS5      /nU(       a&  UR                  S[        R                   " U5      5        [        R"                  " U5      Ul        U(       a[  [&        R"                  " [&        R                  " SS5      [&        R(                  " 5       [&        R*                  " 5       /5      Ul        U$ [&        R"                  " [&        R                  " SS5      /5      Ul        U$ )zç
Similar to convert_from_spm method, but used only when there is no `model_type` class, i.e. there is no matching class in `TOKENIZERS_MAPPING` and we just create a tokenizer instead of extracting stuff from the sentencepiece file
TN)rl   r€   ré   r.  rU  r,  r   )rl   r›   rU  r  õ   â–�)r©   rU  Ú	unk_piecer³   r§   Ú
isinstancery   r   r   rÅ   rÃ   r   r›   r
   ÚReplaceÚinsertÚPrecompiledr]  r   r	   ÚByteFallbackÚFuser	  )r’   rl   r€   rU  r}  r§   r  Ú_normalizerss           rX   Úbuild_tokenizer_from_spm_protoÚ+SpmConverter.build_tokenizer_from_spm_proto  s{  € ð
 ×*Ñ*×8Ñ8ˆØ×&Ñ&×0Ñ0ˆ	Ø$×4Ñ4×IÑIÐô �eœT×"Ñ"Ü!ÜØØ!Ÿ< RØ'Ø!Ø"/Ø ñó	‰Iô ˜œt×$Ñ$®´:¸eÀA¹hÌÔPTÉ×3UÑ3UÜ!ÜØØ ×-Ñ-×4Ñ4Ø"/ñó‰Ið ô $×+Ò+¨C°Ó7Ð8ˆÞØ×Ñ ¤;×#:Ò#:Ð;OÓ#PÔQÜ*×3Ò3°LÓAˆ	Ôö Ü (× 1Ò 1Ü×!Ò! %¨Ó-¬x×/DÒ/DÓ/FÌÏÊËÐXó!ˆIÔð Ðô !)× 1Ò 1´8×3CÒ3CÀEÈ3Ó3OÐ2PÓ QˆIÔàÐrZ   c                 ó   • Ub  XS'   U$ )z‡
Hook used when converting directly from a SentencePiece model without a slow tokenizer instance.
By default, return kwargs unchanged.
rl   rj   )r  rl   r´   s      rX   Úconvert_from_spmÚSpmConverter.convert_from_spm¯  s   € ð ÑØ#�7‰OØˆrZ   c                 óÆ  >• [        U S5        [        TU ]  " U6   [        5       nUR	                  5       n[        U R                  R                  S5       nUR                  UR                  5       5        S S S 5        X0l
        U R                  R                  R                  (       a)  U R                  (       d  [        R                  " S5        g g g ! , (       d  f       Nc= f)NrQ   r�   a  The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.)r   Úsuperr—   rY   rŽ   r�   rc   Ú
vocab_filer�   r‘   r’   r©   rU  Úhandle_byte_fallbackÚwarningsÚwarn)r“   Úargsr”   r•   r–   Ú	__class__s        €rX   r—   ÚSpmConverter.__init__¹  s§   ø€ Ü˜$ 
Ô+ä‰Ò˜$Ñô $Ó%ˆ	à× Ñ Ó"ˆÜ�$×)Ñ)×4Ñ4°dÔ;¸qØ×Ñ˜aŸf™f›hÔ'÷ <àŒ
à�:‰:×"Ñ"×0×0¸×9R×9RÜ�MŠMðeõð :SÐ0÷	 <Õ;ús   Á CÃ
C c                 óp   • UR                    Vs/ s H  o"R                  UR                  4PM     sn$ s  snf rÒ   ©r¬   r­   r®   ©r“   r’   r­   s      rX   rl   ÚSpmConverter.vocabÎ  s)   € Ø8=¿ºÓEº¨u—‘˜eŸk™kÓ*¹ÑEÐEùÒEs   �!3c                 ó.   • UR                   R                  $ rÒ   )r©   r›   ©r“   r’   s     rX   r›   ÚSpmConverter.unk_idÑ  s   € Ø×!Ñ!×(Ñ(Ð(rZ   c                 ó„  • UR                   R                  nU R                  U5      nUS:X  a.  [        [	        UU R                  U5      U R                  S95      nO¡US:X  a�  U R                  U R                  R                  5      R                  U5      u  pV[        U5       VVV	s0 s H
  u  nu  p‰X‡_M     n
nnn	[        [        U
UUR                   R                  SU R                  S S95      nO[        S5      e[        UR                  5       VVs/ s HR  u  p¼UR                   S;   d  M  X¼R"                  UR                   S:H  =(       d    UR"                  U R$                  ;   4PMT     nnnUR'                  [)        US	 S
9 VVVs/ s H  u  p¾n[+        USUS9PM     snnn5        U$ s  sn	nnf s  snnf s  snnnf )Nr   ©r›   rU  rr   T©ré   r.  rU  r,  z]You're trying to run a `Unigram` model but you're file was trained with a different algorithmrœ   r�   c                 ó   • U S   $ r    rj   r¡   s    rX   rm   Ú(SpmConverter.tokenizer.<locals>.<lambda>   ó   € ÐQRÐSTÒQUrZ   ro   Fr£   )r©   r«   rl   r   r   r›   r�  ÚSpmExtractorrc   rŒ  r»   r±   r   r}  Ú	Exceptionr¬   r²   r­   r÷   Ú
add_tokensr}   r   )r“   r’   r«   r   r  Ú_r€   rµ   r¶   r®   Ú	bpe_vocabr·   r¸   r¹   rº   r¥   s                   rX   r  ÚSpmConverter.tokenizerÔ  s®  € Ø×'Ñ'×2Ñ2ˆ
Ø—z‘z %Ó(ˆà˜‹?Ü!ÜØ ØŸ;™; uÓ-Ø"&×";Ñ";ñó‰Ið ˜1‹_Ø×)Ñ)¨$×*AÑ*A×*LÑ*LÓM×UÑUÐVbÓc‰IˆAÜ9BÀ<Ô9PÕQÒ9PÑ%5 Q©¨˜šÑ9PˆIÒQÜ!ÜØØØ#×0Ñ0×:Ñ:Ø!Ø"&×";Ñ";Ø ñó	‰Iô Øoóð ô # 5§<¡<Ô0ô
â0‘�Ø�v‰v˜Ñó IˆR—‘˜!Ÿ&™& A™+×G¨¯©°D×4GÑ4GÑ)GÓHÙ0ð 	ñ 
ð
 	×Ñô +1Ð1AÁ~Ò*Võâ*VÑ&�B˜wô ˜5¨U¸GÔDÙ*Vóô	
ð ÐùôC Rùó*
ùôs   Â'F.ÄF5Ä/?F5ÆF;c                 ó.  • UR                   R                  n[        R                  " SSS9[        R                  " [        S5      S5      /nU(       d  [        R                  " U5      $ [        R                  " [        R                  " U5      /U-   5      $ )NFT)Úleftr  ú {2,}r|  )r³   r§   r
   ÚStripr  r   r]  r�  ©r“   r’   r§   r„  s       rX   r   ÚSpmConverter.normalizer  s{   € Ø$×4Ñ4×IÑIÐä×Ò 5°Ñ5Ü×Ò¤ g£°Ó6ð
ˆö $Ü×'Ò'¨Ó5Ð5ä×'Ò'¬×)@Ò)@ÐAUÓ)VÐ(WÐZfÑ(fÓgÐgrZ   c                 óT   • [        X R                  5      n[        R                  " XS9$ ©N©Úreplacementrd   )re   rc   r   Ú	Metaspace©r“   r¯  r[   rd   s       rX   r  ÚSpmConverter.pre_tokenizer  s$   € Ü,Ð-=×?VÑ?VÓWˆÜ×'Ò'°KÑ_Ð_rZ   c                 ó   • g rÒ   rj   rá   s    rX   r  ÚSpmConverter.post_processor  s   € ØrZ   c                 óT   • [        X R                  5      n[        R                  " XS9$ r­  )re   rc   r	   r°  r±  s       rX   r	  ÚSpmConverter.decoder  s$   € Ü,Ð-=×?VÑ?VÓWˆÜ×!Ò!¨kÑYÐYrZ   r\   c                 óˆ  • U R                  U R                  5      nU R                  U R                  5      nUb  X!l        SnSn[        U R                  S5      (       a  U R                  R
                  nU R                  X45      nUb  XQl        U R                  X45      Ul        U R                  5       nU(       a  Xal        U$ )Nr|  Tr[   )	r  r’   r   rü   rc   r[   r  r	  r  )r“   r  r   r¯  r[   r  r  s          rX   râ   ÚSpmConverter.converted  s¯   € Ø—N‘N 4§:¡:Ó.ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÑ!Ø#-Ô àˆØÐÜ�4×*Ñ*Ð,>×?Ñ?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;ÓIˆØÑ$Ø&3Ô#à ŸL™L¨ÓGˆ	ÔØ×,Ñ,Ó.ˆÞØ'5Ô$àÐrZ   r½   rÒ   )r¯   r¾   r¿   rÀ   r�  r‰   r   r÷   Ústaticmethodr…  Úclassmethodrˆ  r—   rl   r›   r  r   r  r  r	  r   râ   rÆ   Ú__classcell__)r‘  s   @rX   rz  rz  z  sx   ø† Ø ÐØ)€LØ€Nàó-ó ð-ð^ óó ðõò*Fò)ò0òd	hò`òòZð˜9÷ ò rZ   rz  c                   ó&   • \ rS rSrS rS rS rSrg)ÚAlbertConverteri5  c                 óÜ   • UR                    Vs/ s HP  n[        UR                  5      (       a  UR                  UR                  4OUR                  UR                  S-
  4PMR     sn$ s  snf ©Néd   ©r¬   rØ   r­   r®   r•  s      rX   rl   ÚAlbertConverter.vocab6  óf   € ð Ÿšó
â%�ô +=¸U¿[¹[×*IÑ*IˆU�[‰[˜%Ÿ+™+Ñ&ÐPU×P[ÑP[Ð]b×]hÑ]hÐknÑ]nÐOoÒoÙ%ñ
ð 	
ùò 
ó   �AA)c                 ó°  • [         R                  " SS5      [         R                  " SS5      /nU R                  R                  (       dH  UR	                  [         R
                  " 5       5        UR	                  [         R                  " 5       5        U R                  R                  (       a$  UR	                  [         R                  " 5       5        UR                  R                  nU(       a%  UR	                  [         R                  " U5      5        UR	                  [         R                  " [        S5      S5      5        [         R                  " U5      $ ©Nz``Ú"z''r¨  r  ©r
   r  rc   Úkeep_accentsr|   ÚNFKDÚStripAccentsrþ   Ú	Lowercaser³   r§   r�  r   r]  ©r“   r’   Úlist_normalizersr§   s       rX   r   ÚAlbertConverter.normalizer<  óï   € ä×Ò  cÓ*Ü×Ò  cÓ*ð
Ðð ×&Ñ&×3×3Ø×#Ñ#¤K×$4Ò$4Ó$6Ô7Ø×#Ñ#¤K×$<Ò$<Ó$>Ô?Ø×"Ñ"×0×0Ø×#Ñ#¤K×$9Ò$9Ó$;Ô<à$×4Ñ4×IÑIÐæØ×#Ñ#¤K×$;Ò$;Ð<PÓ$QÔRà×Ñ¤× 3Ò 3´E¸'³NÀCÓ HÔIÜ×#Ò#Ð$4Ó5Ð5rZ   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ ©Nrt  ru  rv  rw  rô   ©r   r  rc   r  rá   s    rX   r  ÚAlbertConverter.post_processorO  óR   € Ü×,Ò,Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðñ
ð 	
rZ   rj   N©r¯   r¾   r¿   rÀ   rl   r   r  rÆ   rj   rZ   rX   r½  r½  5  ó   † ò
ò6õ&
rZ   r½  c                   ó    • \ rS rSrS rS rSrg)ÚBarthezConverteriZ  c                 ó
   • SnU$ ©Nr�   rj   ©r“   r’   r›   s      rX   r›   ÚBarthezConverter.unk_id[  ó   € ØˆØˆrZ   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ ©Nz<s> $A </s>z<s> $A </s> </s> $B </s>ú<s>ú</s>rô   rÓ  rá   s    rX   r  ÚBarthezConverter.post_processor_  óR   € Ü×,Ò,Ø Ø+à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðñ
ð 	
rZ   rj   N)r¯   r¾   r¿   rÀ   r›   r  rÆ   rj   rZ   rX   rÙ  rÙ  Z  s   † òõ
rZ   rÙ  c                   ó:   • \ rS rSrS rS rS r\SS j5       rSr	g)	ÚCamembertConverterij  c                 ó’   • / SQnX!R                   SS   Vs/ s H  o3R                  UR                  4PM     sn-  nUS/-  nU$ s  snf )N)©z
<s>NOTUSEDç        ©ú<pad>ré  ©z</s>NOTUSEDré  ©ú<unk>ré  )ú<unk>NOTUSEDiœÿÿÿr   ©ú<mask>ré  r”  ©r“   r’   rl   r­   s       rX   rl   ÚCamembertConverter.vocabk  sR   € ò
ˆð 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØ�/Ð"Ñ"ˆØˆùò Ló   –!Ac                 ó   • grÛ  rj   r˜  s     rX   r›   ÚCamembertConverter.unk_idx  s   € àrZ   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ rà  rÓ  rá   s    rX   r  Ú!CamembertConverter.post_processor|  rä  rZ   Nc                 ó,  • [        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      nSUS4S	US4S
/nUb  UR                  [        U5      SS  5        UR	                  US45        XbS'   U$ )NÚ	pad_tokenrë  ré   rî  Ú
mask_tokenrñ  rè  ré  rì  )rï  ç      YÀr   rl   ©rÂ   rÐ   r~   rÅ   r|   )r  rl   r´   rú  ré   rû  Ú
vocab_lists          rX   rˆ  Ú#CamembertConverter.convert_from_spm†  s¢   € ä˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð  Ø˜ÐØ Ø˜ÐØ$ð
ˆ
ð ÑØ×Ñœd 5›k¨!¨"˜oÔ.Ø×Ñ˜: sÐ+Ô,Ø$ˆw‰ØˆrZ   rj   rÒ   ©
r¯   r¾   r¿   rÀ   rl   r›   r  rº  rˆ  rÆ   rj   rZ   rX   ræ  ræ  j  s%   † òòò
ð óó órZ   ræ  c                   ó&   • \ rS rSrS rS rS rSrg)ÚDebertaV2Converteriš  c                 ó  • / nU R                   R                  (       a#  UR                  [        R                  " SS95        [        X R                   5      nUR                  [        R                  " XS95        [        R                  " U5      $ )NrV  )rX  r®  )rc   Úsplit_by_punctr|   r   ÚPunctuationre   r°  r]  )r“   r¯  r[   Úlist_pretokenizersrd   s        rX   r  Ú DebertaV2Converter.pre_tokenizer›  sl   € ØÐØ×"Ñ"×1×1Ø×%Ñ%¤n×&@Ò&@È*Ñ&UÔVÜ,Ð-=×?VÑ?VÓWˆØ×!Ñ!¤.×":Ò":À{Ñ"rÔsÜ×&Ò&Ð'9Ó:Ð:rZ   c                 óÚ  • / nU R                   R                  (       a$  UR                  [        R                  " 5       5        UR                  [        R
                  " 5       5        UR                  R                  nU(       a%  UR                  [        R                  " U5      5        UR                  [        R                  " [        S5      S5      5        [        R                  " U5      $ )Nr¨  r  )rc   rþ   r|   r
   rÌ  r©  r³   r§   r�  r  r   r]  rÍ  s       rX   r   ÚDebertaV2Converter.normalizer£  s¥   € ØÐØ×"Ñ"×0×0Ø×#Ñ#¤K×$9Ò$9Ó$;Ô<Ø×Ñ¤× 1Ò 1Ó 3Ô4à$×4Ñ4×IÑIÐÞØ×#Ñ#¤K×$;Ò$;Ð<PÓ$QÔRØ×Ñ¤× 3Ò 3´E¸'³NÀCÓ HÔIä×#Ò#Ð$4Ó5Ð5rZ   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ rÒ  rÓ  rá   s    rX   r  Ú!DebertaV2Converter.post_processor°  rÕ  rZ   rj   N)r¯   r¾   r¿   rÀ   r  r   r  rÆ   rj   rZ   rX   r  r  š  s   † ò;ò6õ
rZ   r  c                   ó:   • \ rS rSrS rS rS r\SS j5       rSr	g)	ÚMBartConverteri»  c                 ó    • / SQnX!R                   SS   Vs/ s H  o3R                  UR                  4PM     sn-  nU/ SQ-  nUS/-  nU$ s  snf )N©©rá  ré  rê  ©râ  ré  rí  r�   )©r   ré  ©r   ré  ©r   ré  ©r   ré  ©r   ré  ©r   ré  ©r   ré  ©r   ré  ©r   ré  ©r    ré  ©r!   ré  ©r"   ré  ©r#   ré  ©r$   ré  ©r%   ré  ©r&   ré  ©r'   ré  ©r(   ré  ©r)   ré  ©r*   ré  ©r+   ré  ©r,   ré  ©r-   ré  ©r.   ré  ©r/   ré  rð  r”  rò  s       rX   rl   ÚMBartConverter.vocab¼  sc   € ò
ˆð 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØò 
ñ 	
ˆð6 	�/Ð"Ñ"ˆØˆùò; Ló   –!Ac                 ó   • grÛ  rj   r˜  s     rX   r›   ÚMBartConverter.unk_idâ  ó   € ØrZ   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ )Nz$A </s> en_XXz$A $B </s> en_XXr   râ  rô   rÓ  rá   s    rX   r  ÚMBartConverter.post_processorå  óR   € Ü×,Ò,Ø"Ø#à˜$×1Ñ1×GÑGÈÓPÐQØ˜×0Ñ0×FÑFÀvÓNÐOðñ
ð 	
rZ   Nc                 óÖ  • [        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  S	S
5      5      nUS4US4US4US4/nUb  UR                  [        U5      SS  5        UR                  S [         5       5        UR                  US45        X‚S'   U$ )NrD  rá  rú  rë  Ú	eos_tokenrâ  ré   rî  rû  rñ  ré  r�   c              3   ó(   #   • U  H  oS 4v •  M
     g7f©ré  Nrj   ©Ú.0Ú	lang_codes     rX   Ú	<genexpr>Ú2MBartConverter.convert_from_spm.<locals>.<genexpr>ÿ  s   é € ÐLºO¨y cÕ*ºOùó   ‚rl   )rÂ   rÐ   r~   rÅ   ÚMBART_LANGUAGESr|   ©	r  rl   r´   rD  rú  r4  ré   rû  rþ  s	            rX   rˆ  ÚMBartConverter.convert_from_spmï  sç   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÑØ×Ñœd 5›k¨!¨"˜oÔ.Ø×ÑÑL½OÓLÔLØ×Ñ˜: sÐ+Ô,Ø$ˆw‰ØˆrZ   rj   rÒ   r   rj   rZ   rX   r  r  »  s&   † ò$òLò
ð óó órZ   r  c                   ó:   • \ rS rSrS rS rS r\SS j5       rSr	g)	ÚMBart50Converteri  c                 ó    • / SQnX!R                   SS   Vs/ s H  o3R                  UR                  4PM     sn-  nU/ SQ-  nUS/-  nU$ s  snf )Nr  r�   )4r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r!  r"  r#  r$  r%  r&  r'  r(  r)  r*  )r0   ré  )r1   ré  )r2   ré  )r3   ré  )r4   ré  )r5   ré  )r6   ré  )r7   ré  )r8   ré  )r9   ré  )r:   ré  )r;   ré  )r<   ré  )r=   ré  )r>   ré  )r?   ré  )r@   ré  )rA   ré  )rB   ré  )rC   ré  )rD   ré  )rE   ré  )rF   ré  )rG   ré  )rH   ré  )rI   ré  )rJ   ré  rð  r”  rò  s       rX   rl   ÚMBart50Converter.vocab  sc   € ò
ˆð 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØò  Rñ  	RˆØ�/Ð"Ñ"ˆØˆùò Lr,  c                 ó   • grÛ  rj   r˜  s     rX   r›   ÚMBart50Converter.unk_id  r/  rZ   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ )Nzen_XX $A </s>zen_XX $A $B </s>r   râ  rô   rÓ  rá   s    rX   r  ÚMBart50Converter.post_processor  r2  rZ   Nc                 óÖ  • [        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  S	S
5      5      nUS4US4US4US4/nUb  UR                  [        U5      SS  5        UR                  S [         5       5        UR                  US45        X‚S'   U$ )Nr  rá  rú  rë  r4  râ  ré   rî  rû  rñ  ré  r�   c              3   ó(   #   • U  H  oS 4v •  M
     g7fr6  rj   r7  s     rX   r:  Ú4MBart50Converter.convert_from_spm.<locals>.<genexpr>/  s   é € ÐNÒ<M¨y cÕ*Ò<Mùr<  rl   )rÂ   rÐ   r~   rÅ   ÚMBART50_LANGUAGESr|   )	r  rl   r´   r  rú  r4  ré   rû  rþ  s	            rX   rˆ  Ú!MBart50Converter.convert_from_spm  sè   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÑØ×Ñœd 5›k¨!¨"˜oÔ.Ø×ÑÑNÕ<MÓNÔNØ×Ñ˜: sÐ+Ô,Ø$ˆw‰ØˆrZ   rj   rÒ   r   rj   rZ   rX   rA  rA    s%   † ò
òò
ð óó órZ   rA  c                   ó:   • \ rS rSrS rS rS r\SS j5       rSr	g)	ÚNllbConverteri5  c                 ó†   • / SQnX!R                   SS   Vs/ s H  o3R                  UR                  4PM     sn-  nU$ s  snf )Nr  r�   r”  rò  s       rX   rl   ÚNllbConverter.vocab6  óE   € ò
ˆð 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØˆùò Ló   –!>c                 ó   • grÛ  rj   r˜  s     rX   r›   ÚNllbConverter.unk_id@  r/  rZ   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ )Nzeng_Latn $A </s>zeng_Latn $A $B </s>Úeng_Latnrâ  rô   rÓ  rá   s    rX   r  ÚNllbConverter.post_processorC  sR   € Ü×,Ò,Ø%Ø&à˜T×4Ñ4×JÑJÈ:ÓVÐWØ˜×0Ñ0×FÑFÀvÓNÐOðñ
ð 	
rZ   Nc                 ó¾  • [        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      nUS	US
USUS0nUbY  [        U[        5      (       a  UR	                  5       OU VV	s/ s H  u  p‰UPM	     sn	nn
U
 H  nX·;   a  M
  [        U5      X{'   M     XrS'   U$ s  sn	nf )NrD  rá  rú  rë  r4  râ  ré   rî  r   r   rr   r�   rl   )rÂ   rÐ   r~  ry   r3  rs   )r  rl   r´   rD  rú  r4  ré   Úreordered_vocabÚtokr£  Útokensrº   s               rX   rˆ  ÚNllbConverter.convert_from_spmM  sÝ   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	ð �qØ�qØ�qØ�qð	
ˆð ÑÜ%/°´t×%<Ñ%<�U—Z‘Z”\ÑUZÔB[ÒUZÉ6È3Ã3ÑUZÒB[ˆFÛ�ØÓ+ÙÜ),¨_Ó)=�Ó&ñ  ð *ˆw‰Øˆùó C\s   Â$Crj   rÒ   r   rj   rZ   rX   rN  rN  5  s%   † òòò
ð óó órZ   rN  c                   ó&   • \ rS rSrS rS rS rSrg)ÚSeamlessM4TConverterid  c                 ó†   • / SQnX!R                   SS   Vs/ s H  o3R                  UR                  4PM     sn-  nU$ s  snf )N)rê  rí  r  r  r�   r”  rò  s       rX   rl   ÚSeamlessM4TConverter.vocabe  rQ  rR  c                 ó.   • U R                   R                  $ rÒ   )rc   Úunk_token_idr˜  s     rX   r›   ÚSeamlessM4TConverter.unk_ido  s   € Ø×&Ñ&×3Ñ3Ð3rZ   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ )Nz__eng__ $A </s>z__eng__ $A $B </s>Ú__eng__râ  rô   rÓ  rá   s    rX   r  Ú#SeamlessM4TConverter.post_processorr  sR   € Ü×,Ò,Ø$Ø%à˜D×3Ñ3×IÑIÈ)ÓTÐUØ˜×0Ñ0×FÑFÀvÓNÐOðñ
ð 	
rZ   rj   N©r¯   r¾   r¿   rÀ   rl   r›   r  rÆ   rj   rZ   rX   r^  r^  d  s   † òò4õ
rZ   r^  c                   ó:   • \ rS rSrS rS rS r\SS j5       rSr	g)	ÚXLMRobertaConverteri}  c                 ó’   • / SQnX!R                   SS   Vs/ s H  o3R                  UR                  4PM     sn-  nUS/-  nU$ s  snf )Nr  r�   rð  r”  rò  s       rX   rl   ÚXLMRobertaConverter.vocab~  sR   € ò
ˆð 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØ�/Ð"Ñ"ˆØˆùò Lrô  c                 ó
   • SnU$ rÛ  rj   rÜ  s      rX   r›   ÚXLMRobertaConverter.unk_id‰  rÞ  rZ   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ rà  rÓ  rá   s    rX   r  Ú"XLMRobertaConverter.post_processor�  rä  rZ   Nc                 óž  • [        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  S	S
5      5      nUS4US4US4US4/nUb  UR                  [        U5      SS  5        UR	                  US45        X‚S'   U$ )NrD  rá  rú  rë  r4  râ  ré   rî  rû  rñ  ré  r�   rl   rý  r>  s	            rX   rˆ  Ú$XLMRobertaConverter.convert_from_spm—  sÓ   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÑØ×Ñœd 5›k¨!¨"˜oÔ.Ø×Ñ˜: sÐ+Ô,Ø$ˆw‰ØˆrZ   rj   rÒ   r   rj   rZ   rX   ri  ri  }  s%   † ò	òò
ð óó órZ   ri  c                   ó&   • \ rS rSrS rS rS rSrg)ÚXLNetConverteri¬  c                 óÜ   • UR                    Vs/ s HP  n[        UR                  5      (       a  UR                  UR                  4OUR                  UR                  S-
  4PMR     sn$ s  snf r¿  rÁ  r•  s      rX   rl   ÚXLNetConverter.vocab­  rÃ  rÄ  c                 ó°  • [         R                  " SS5      [         R                  " SS5      /nU R                  R                  (       dH  UR	                  [         R
                  " 5       5        UR	                  [         R                  " 5       5        U R                  R                  (       a$  UR	                  [         R                  " 5       5        UR                  R                  nU(       a%  UR	                  [         R                  " U5      5        UR	                  [         R                  " [        S5      S5      5        [         R                  " U5      $ rÆ  rÈ  rÍ  s       rX   r   ÚXLNetConverter.normalizer³  rÐ  rZ   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ )Nz$A:0 <sep>:0 <cls>:2z!$A:0 <sep>:0 $B:1 <sep>:1 <cls>:2z<sep>z<cls>rô   rÓ  rá   s    rX   r  ÚXLNetConverter.post_processorÆ  rÕ  rZ   rj   NrÖ  rj   rZ   rX   rs  rs  ¬  r×  rZ   rs  c                   ó   • \ rS rSrSrg)ÚReformerConverteriÑ  rj   N©r¯   r¾   r¿   rÀ   rÆ   rj   rZ   rX   r{  r{  Ñ  ó   † ÚrZ   r{  c                   ó    • \ rS rSrS rS rSrg)ÚRemBertConverteriÕ  c                 ó�  • [         R                  " SS5      [         R                  " SS5      [         R                  " [        S5      S5      /nU R                  R                  (       dH  UR                  [         R                  " 5       5        UR                  [         R                  " 5       5        U R                  R                  (       a$  UR                  [         R                  " 5       5        UR                  R                  nU(       a%  UR                  [         R                  " U5      5        [         R                  " U5      $ rÆ  )r
   r  r   rc   rÉ  r|   rÊ  rË  rþ   rÌ  r³   r§   r�  r]  rÍ  s       rX   r   ÚRemBertConverter.normalizer×  sä   € ä×Ò  cÓ*Ü×Ò  cÓ*Ü×Ò¤ g£°Ó4ð
Ðð
 ×&Ñ&×3×3Ø×#Ñ#¤K×$4Ò$4Ó$6Ô7Ø×#Ñ#¤K×$<Ò$<Ó$>Ô?Ø×"Ñ"×0×0Ø×#Ñ#¤K×$9Ò$9Ó$;Ô<à$×4Ñ4×IÑIÐæØ×#Ñ#¤K×$;Ò$;Ð<PÓ$QÔRä×#Ò#Ð$4Ó5Ð5rZ   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ rÒ  rÓ  rá   s    rX   r  ÚRemBertConverter.post_processorê  rÕ  rZ   rj   N)r¯   r¾   r¿   rÀ   r   r  rÆ   rj   rZ   rX   r  r  Õ  s   † ò6õ&
rZ   r  c                   ó   • \ rS rSrSrg)ÚBertGenerationConverteriõ  rj   Nr|  rj   rZ   rX   r…  r…  õ  r}  rZ   r…  c                   ó@   • \ rS rSrS r\S	S j5       rS rS rS r	Sr
g)
ÚPegasusConverteriù  c                 ó€  • U R                   R                  S4U R                   R                  S4/nU R                   R                  b  X R                   R                  S4/-  nU R                   R                  bI  U R                   R
                  U R                   R                  :  a  X R                   R                  S4/-  nU[        SU R                   R                  5       Vs/ s H  nSU S3S4PM     sn-  nX!R                  SS   Vs/ s H  oDR                  UR                  4PM     sn-  nU$ s  snf s  snf )Nré  rr   ú<unk_Ú>rü  )rc   rú  r4  Úmask_token_sentrû  Úmask_token_idÚoffsetr{   r¬   r­   r®   )r“   r’   rl   rµ   r­   s        rX   rl   ÚPegasusConverter.vocabú  s)  € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 ×"Ñ"×2Ñ2Ñ>Ø×.Ñ.×>Ñ>ÀÐDÐEÑEˆEð ×#Ñ#×.Ñ.Ñ:Ø×'Ñ'×5Ñ5¸×8OÑ8O×8VÑ8VÓVà×.Ñ.×9Ñ9¸3Ð?Ð@Ñ@ˆEà´%¸¸4×;RÑ;R×;YÑ;YÔ2ZÓ[Ò2Z¨Q�U˜1˜#˜Q�< Ó(Ñ2ZÑ[Ñ[ˆØ¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØˆùò \ùÚKs   Ã&D6Ä!D;Nc           	      ó2  • [        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      n[        UR                  SS5      5      nUS	4US	4/nUS
:w  a  UR                  US	45        US
:w  a  Xe:w  a  UR                  US	45        UR                  [	        SUR                  SS5      5       Vs/ s H  nSU S3S4PM     sn5        Ub  UR                  [        U5      SS  5        XrS'   U$ s  snf )Nrú  rë  r4  râ  rû  z<mask_1>r‹  z<mask_2>ré  ÚNonerr   r�  ég   r‰  rŠ  rü  rl   )rÂ   rÐ   r|   r~   r{   rÅ   )	r  rl   r´   rú  r4  rû  r‹  rþ  rµ   s	            rX   rˆ  Ú!PegasusConverter.convert_from_spm  s  € ä˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ™ L°*Ó=Ó>ˆ
Ü˜fŸj™jÐ):¸JÓGÓHˆð ˜ÐØ˜Ðð
ˆ
ð ˜ÓØ×Ñ˜z¨3Ð/Ô0Ø˜fÓ$¨Ó)FØ×Ñ˜°Ð4Ô5à×Ñ¼5ÀÀFÇJÁJÈxÐY\ÓD]Ô;^Ó_Ò;^°a˜e A 3 a˜L¨&Ó1Ñ;^Ñ_Ô`ØÑØ×Ñœd 5›k¨!¨"˜oÔ.Ø$ˆw‰Øˆùò	 `s   ÃDc                 ó\   • UR                   R                  U R                  R                  -   $ rÒ   )r©   r›   rc   r�  r˜  s     rX   r›   ÚPegasusConverter.unk_id#  s%   € Ø×!Ñ!×(Ñ(¨4×+BÑ+B×+IÑ+IÑIÐIrZ   c                 ó¦   • [        X R                  5      n[        R                  " [        R                  " 5       [        R
                  " XS9/5      $ r­  )re   rc   r   r]  ÚWhitespaceSplitr°  r±  s       rX   r  ÚPegasusConverter.pre_tokenizer&  sE   € Ü,Ð-=×?VÑ?VÓWˆÜ×&Ò&ä×.Ò.Ó0Ü×(Ò(°[Ñ`ðó
ð 	
rZ   c                 ó”   • U R                   R                  nXR                   R                  4/n[        R                  " SU/SSU/US9$ )Nú$Aú$Brô   )rc   r4  Úeos_token_idr   r  )r“   Úeosr÷   s      rX   r  ÚPegasusConverter.post_processor/  sP   € Ø×%Ñ%×/Ñ/ˆà×)Ñ)×6Ñ6Ð7ð
ˆô ×,Ò,°T¸3°KÀtÈTÐSVÐFWÐhvÑwÐwrZ   rj   rÒ   )r¯   r¾   r¿   rÀ   rl   rº  rˆ  r›   r  r  rÆ   rj   rZ   rX   r‡  r‡  ù  s,   † òð& óó ðò*Jò
õxrZ   r‡  c                   ó4   • \ rS rSrS rS r\SS j5       rSrg)ÚT5Converteri7  c                 óü   • U R                   R                  nUR                   Vs/ s H  o3R                  UR                  4PM     nnU[        US-
  SS5       Vs/ s H  nSU S3S4PM     sn-  nU$ s  snf s  snf )Nr   rÔ   ú
<extra_id_rŠ  ré  )rc   Ú
_extra_idsr¬   r­   r®   r{   )r“   r’   Únum_extra_idsr­   rl   rµ   s         rX   rl   ÚT5Converter.vocab8  s|   € Ø×/Ñ/×:Ñ:ˆØ9>¿ºÓFº°—+‘+˜uŸ{™{Ó+¹ˆÐFØ´E¸-È!Ñ:KÈRÐQSÔ4TÓUÒ4T¨q�Z ˜s !Ð$ cÓ*Ñ4TÑUÑUˆØˆùò GùÚUs   ¥!A4ÁA9c                 ón   • [         R                  " SS// SQSU R                  R                  S5      4/S9$ ©Nr™  râ  )r™  râ  rš  râ  rô   rÓ  rá   s    rX   r  ÚT5Converter.post_processor>  ó=   € Ü×,Ò,Ø˜&�>Ú-à˜×0Ñ0×FÑFÀvÓNÐOðñ
ð 	
rZ   Nc                 óø   • UR                  SS5      n[        US-
  SS5       Vs/ s H	  nSU S3PM     nnUb  [        U5      O/ nUR                  S U 5       5        UR	                  SU5        XbS	'   U$ s  snf )
NÚ	extra_idsrÀ  r   rÔ   r¡  rŠ  c              3   ó(   #   • U  H  oS 4v •  M
     g7fr6  rj   )r8  rº   s     rX   r:  Ú/T5Converter.convert_from_spm.<locals>.<genexpr>L  s   é € ÐA²L¨5 #�,²Lùr<  r¦   rl   )rÐ   r{   rÅ   r~   Ú
setdefault)r  rl   r´   rª  rµ   Úextra_tokensrþ  s          rX   rˆ  ÚT5Converter.convert_from_spmG  s…   € à—J‘J˜{¨CÓ0ˆ	Ü38¸ÀQ¹ÈÈBÔ3OÓPÒ3O¨a˜* Q C qÓ)Ñ3OˆÐPØ$)Ñ$5”T˜%”[¸2ˆ
Ø×ÑÑA±LÓAÔAà×ÑÐ5°|ÔDØ$ˆw‰Øˆùò Qs   ¥A7rj   rÒ   )	r¯   r¾   r¿   rÀ   rl   r  rº  rˆ  rÆ   rj   rZ   rX   rŸ  rŸ  7  s    † òò
ð óó órZ   rŸ  c                   ó   • \ rS rSrS rSrg)ÚUdopConverteriS  c                 ón   • [         R                  " SS// SQSU R                  R                  S5      4/S9$ r¦  rÓ  rá   s    rX   r  ÚUdopConverter.post_processorT  r¨  rZ   rj   N©r¯   r¾   r¿   rÀ   r  rÆ   rj   rZ   rX   r±  r±  S  s   † õ
rZ   r±  c                   ó"   • \ rS rSrS\4S jrSrg)ÚWhisperConverteri^  r\   c                 ó¼  • U R                   R                  n[        U R                   R                  R	                  5       5      n[        [        UUS SSSS95      n[        R                  " U R                   R                  S9Ul
        [        R                  " 5       Ul        U R                   R                  nU R                   R                  U5      nU R                   R                  nU R                   R                   nSR#                  U Vs/ s H  oˆ S3PM	     sn5      n	[$        R&                  " U	 SU S3U	 SU S	3Xg4/[)        XT5      QS
9Ul        U$ s  snf )Nr<  Fr=  r?  r  rñ   z $A:0 z $A:0 $B:1 ró   rô   )rc   r1  rÅ   r2  r3  r   r   r   rC  r[   r  r	   r	  Úprefix_tokensÚconvert_ids_to_tokensr4  r›  Újoinr   r  Úzipr  )
r“   rl   r€   r  Úprefix_token_idsÚprefixesrœ  r›  rº   Úprefix_templates
             rX   râ   ÚWhisperConverter.converted_  sO  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆäÜØØØØ*,Ø#%Øñó	
ˆ	ô #1×":Ò":ÈD×LcÑLc×LtÑLtÑ"uˆ	ÔÜ$×.Ò.Ó0ˆ	Ôà×2Ñ2×@Ñ@ÐØ×*Ñ*×@Ñ@ÐAQÓRˆØ×%Ñ%×/Ñ/ˆØ×.Ñ.×;Ñ;ˆØŸ(™(¹hÓ#Gºh°U g¨R£L¹hÑ#GÓHˆÜ#-×#@Ò#@Ø%Ð& f¨S¨E°Ð4Ø#Ð$ K°¨u°BÐ7àÐ#ðä�XÓ0ðñ$
ˆ	Ô ð Ðùò $Hs   ÄErj   Nr  rj   rZ   rX   r¶  r¶  ^  s   † ð ˜9÷  rZ   r¶  c                   ó   • \ rS rSrS rSrg)ÚBigBirdConverteri‚  c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ rÒ  rÓ  rá   s    rX   r  ÚBigBirdConverter.post_processorƒ  rÕ  rZ   rj   Nr´  rj   rZ   rX   rÁ  rÁ  ‚  s   † õ
rZ   rÁ  c                   ó"   • \ rS rSrS\4S jrSrg)ÚCLIPConverteriŽ  r\   c                 ój  • U R                   R                  n[        U R                   R                  R	                  5       5      nU R                   R
                  n[        [        UUS SSS[        U5      S95      n[        R                  " [        R                  " 5       [        R                  " [        S5      S5      [        R                  " 5       /5      Ul        [         R                  " [         R"                  " [        S5      SS	S
9[         R$                  " SS9/5      Ul        [(        R$                  " 5       Ul        [,        R.                  " U R                   R0                  U R                   R2                  4U R                   R4                  U R                   R6                  4SSS9Ul        U$ )Nr<  r+  F©rl   r€   r,  r>  r-  r.  ré   z\s+r  z9's|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+ÚremovedTrW  r?  rc  )rc   r1  rÅ   r2  r3  ré   r   r   rÂ   r
   r]  r\  r  r   rÌ  r   r   r^  rC  r  r	   r	  r   rd  r4  r›  rD  rE  r  r7  s        rX   râ   ÚCLIPConverter.converted�  sk  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØØ*,Ø#)ØÜ˜i›.ñó

ˆ	ô  +×3Ò3Ü�_Š_Ó¤× 3Ò 3´E¸&³MÀ3Ó GÌ×I^ÒI^ÓI`Ðaó 
ˆ	Ôô #1×"9Ò"9ä×$Ò$ÜÐZÓ[Ø&Øñô
 ×(Ò(¸%Ñ@ðó	#
ˆ	Ôô %×.Ò.Ó0ˆ	Ôô $.×#?Ò#?Ø×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ"Øñ	$
ˆ	Ô ð ÐrZ   rj   Nr  rj   rZ   rX   rÅ  rÅ  Ž  s   † ð'˜9÷ 'rZ   rÅ  c                   ó"   • \ rS rSrS\4S jrSrg)ÚLayoutLMv2Converteri¹  r\   c           	      ób  • U R                   R                  n[        [        U[	        U R                   R
                  5      S95      nSnSnSn[        U R                   S5      (       a`  U R                   R                  R                  nU R                   R                  R                  nU R                   R                  R                  n[        R                  " SUUUS9Ul        [        R                  " 5       Ul        [	        U R                   R"                  5      n[	        U R                   R$                  5      nU R                   R&                  nU R                   R(                  n	[*        R,                  " U SU S3U SU SU S	3Xh4Xy4/S
9Ul        [0        R                  " SS9Ul        U$ )Nrè   FTrê   rë   rð   rñ   rò   ró   rô   rø   rù   rû   r
  s
             rX   râ   ÚLayoutLMv2Converter.convertedº  s„  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ñ`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ñ>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ñ	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5àÐ#ØÐ#ðñ$
ˆ	Ô ô %×.Ò.°dÑ;ˆ	ÔàÐrZ   rj   Nr  rj   rZ   rX   rË  rË  ¹  r  rZ   rË  c                   ó"   • \ rS rSrS\4S jrSrg)ÚBlenderbotConverterià  r\   c                 ó¢  • U R                   nUR                  n[        UR                  R	                  5       5      n[        [        UUS SSSS95      n[        R                  " UR                  S9Ul
        [        R                  " 5       Ul        [        R                  " SUR                   S3UR                  UR                   4/S9Ul        U$ )Nr<  Fr=  r?  z$A:0 rñ   )rõ   r÷   )rc   r1  rÅ   r2  r3  r   r   r   rC  r[   r  r	   r	  r   r  r4  r›  r  re  s        rX   râ   ÚBlenderbotConverter.convertedá  s½   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øñó	
ˆ	ô #1×":Ò":ÈB×L_ÑL_Ñ"`ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ#-×#@Ò#@Ø˜2Ÿ<™<˜.¨Ð+à—‘˜rŸ™Ð/ðñ$
ˆ	Ô ð ÐrZ   rj   Nr  rj   rZ   rX   rÏ  rÏ  à  rh  rZ   rÏ  c                   ó&   • \ rS rSrS rS rS rSrg)ÚXGLMConverteriý  c                 ó”   • / SQnX!R                   SS   Vs/ s H  o3R                  UR                  4PM     sn-  nU/ SQ-  nU$ s  snf )Nr  r�   ))z<madeupword0>ré  )z<madeupword1>ré  )z<madeupword2>ré  )z<madeupword3>ré  )z<madeupword4>ré  )z<madeupword5>ré  )z<madeupword6>ré  r”  rò  s       rX   rl   ÚXGLMConverter.vocabþ  sV   € ò
ˆð 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØò  zñ  	zˆØˆùò Ls   –!Ac                 ó
   • SnU$ rÛ  rj   rÜ  s      rX   r›   ÚXGLMConverter.unk_id	  rÞ  rZ   c           	      óž   • [         R                  " SSSU R                  R                  S5      4SU R                  R                  S5      4/S9$ )Nz</s> $Az</s> $A </s> </s> $Brá  râ  rô   rÓ  rá   s    rX   r  ÚXGLMConverter.post_processor  sR   € Ü×,Ò,ØØ'à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðñ
ð 	
rZ   rj   Nrg  rj   rZ   rX   rÓ  rÓ  ý  s   † ò	òõ
rZ   rÓ  c                   óD   • \ rS rSrSr\rSS1r S rS r	S r
S rS	 rS
rg)ÚGemmaConverteri  Tz<start_of_turn>z<end_of_turn>c                 ó0   • [         R                  " SS5      $ ©Nr  r|  )r
   r  r˜  s     rX   r   ÚGemmaConverter.normalizer(  s   € Ü×"Ò" 3¨Ó.Ð.rZ   c                 ó€  • U R                   R                  S4U R                   R                  S4U R                   R                  S4/nX!R                  SS   Vs/ s H  o3R
                  UR                  4PM     sn-  n[        S U 5       5      (       d#  [        S [        U5       5       S 5      nUb  SX$'   U$ s  snf )Nré  r�   c              3   ó0   #   • U  H  oS    S:H  v •  M     g7f)r   rË   Nrj   )r8  rk   s     rX   r:  Ú'GemmaConverter.vocab.<locals>.<genexpr>4  s   é € Ð/ª A�Q‘4˜4–<ªùs   ‚c              3   ó@   #   • U  H  u  pUS    S:X  d  M  Uv •  M     g7f)r   rÌ   Nrj   )r8  rµ   rk   s      rX   r:  rá  5  s!   é € Ð"VÒ1A©¨ÀQÀqÁTÈXÑEU§1¡1Ò1Aùs   ‚•	)rË   ré  )
rc   rú  r4  rD  r¬   r­   r®   ÚanyÚnextr±   )r“   r’   rl   r­   Úoverride_indexs        rX   rl   ÚGemmaConverter.vocab+  sº   € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆô Ñ/©Ó/×/Ñ/Ü!Ñ"V´¸5Ô1AÓ"VÐX\Ó]ˆNØÑ)Ø(3�Ñ%àˆùò Ls   Á!B;c                 ó0   • [         R                  " SS5      $ )Nr  Úmerged_with_previous)r   r^  ©r“   r¯  r[   s      rX   r  ÚGemmaConverter.pre_tokenizer;  s   € Ü×#Ò# CÐ)?Ó@Ð@rZ   c                 ó
   • SnU$ rÛ  rj   rÜ  s      rX   r›   ÚGemmaConverter.unk_id>  rÞ  rZ   c                 óª   • [         R                  " [         R                  " SS5      [         R                  " 5       [         R                  " 5       /5      $ )Nr|  r  )r	   r]  r  r‚  rƒ  ré  s      rX   r	  ÚGemmaConverter.decoderB  s?   € Ü× Ò ä× Ò  ¨Ó,Ü×%Ò%Ó'Ü—’“ðó
ð 	
rZ   rj   N)r¯   r¾   r¿   rÀ   r�  rÈ   r   r÷   r   rl   r  r›   r	  rÆ   rj   rZ   rX   rÛ  rÛ    s6   † ØÐØ.€Là'¨Ð9€Nðò/òò Aòõ
rZ   rÛ  c                   ó<   • \ rS rSrSrS rS rS rS rS r	S r
S	rg
)ÚLlamaConverteriL  Tc                 ó*  • U R                   R                  S5      S4U R                   R                  S5      S4U R                   R                  S5      S4/nX!R                  SS   Vs/ s H  o3R                  UR                  4PM     sn-  nU$ s  snf )Nr   ré  r   rr   r�   )rc   r¹  r¬   r­   r®   rò  s       rX   rl   ÚLlamaConverter.vocabO  s�   € à×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCð
ˆð
 	¿,¹,ÀqÀrÑ:JÓKÒ:J°—;‘; §¡Ó,Ñ:JÑKÑKˆØˆùò Ls   Á(!Bc                 ó
   • SnU$ r    rj   rÜ  s      rX   r›   ÚLlamaConverter.unk_idX  rÞ  rZ   c                 óî   • [         R                  " SS5      [         R                  " 5       [         R                  " 5       /nU(       a  U[         R                  " SSS9/-  n[         R
                  " U5      $ ©Nr|  r  r   )Úcontentr§  ©r	   r  r‚  rƒ  r©  r]  ©r“   r¯  r[   Úsequences       rX   r	  ÚLlamaConverter.decoder\  ó\   € ä×Ò˜U CÓ(Ü×!Ò!Ó#Ü�MŠM‹Oð
ˆö
 ØœŸš°¸!Ñ<Ð=Ñ=ˆHÜ× Ò  Ó*Ð*rZ   c                 ó  • [        U R                  SS5      (       ae  / n[        U R                  SS5      (       a  U[        R                  " SS9/-  nU[        R                  " SSS9/-  n[        R
                  " U5      $ g )Nr_   Tr[   r|  )Úprependr  )Úpatternr÷  )rb   rc   r
   ÚPrependr  r]  )r“   r’   rú  s      rX   r   ÚLlamaConverter.normalizerf  sx   € Ü�4×*Ñ*¨H°d×;Ñ;ØˆHÜ�t×.Ñ.Ð0BÀD×IÑIØœ[×0Ò0¸Ñ?Ð@Ñ@�Øœ×,Ò,°SÀ%ÑHÐIÑIˆHÜ×'Ò'¨Ó1Ð1ØrZ   c                 ó�   • [        U R                  SS5      (       d*  [        X R                  5      n[        R                  " XSS9$ g )Nr_   TF©r¯  rd   Úsplit)rb   rc   re   r   r°  r±  s       rX   r  ÚLlamaConverter.pre_tokenizero  s?   € Ü�t×.Ñ.°¸$×?Ñ?Ü0Ð1A×CZÑCZÓ[ˆNÜ!×+Ò+¸ÐjoÑpÐpØrZ   c                 ó   • g rÒ   rj   rá   s    rX   r  ÚLlamaConverter.post_processoru  s   € àrZ   rj   N)r¯   r¾   r¿   rÀ   r�  rl   r›   r	  r   r  r  rÆ   rj   rZ   rX   rð  rð  L  s&   † ØÐòòò+òòõrZ   rð  c                   ó"   • \ rS rSrS\4S jrSrg)ÚMarkupLMConverteriz  r\   c                 óz  • U R                   nUR                  n[        UR                  R	                  5       5      n[        [        UUS SSSU R                   R                  S95      n[        R                  " UR                  S9Ul        [        R                  " 5       Ul        [        U R                   R                  5      n[        U R                   R                   5      nU R                   R"                  nU R                   R$                  n[&        R(                  " U SU 3U SU SU 3XW4Xh4/S9Ul        U$ )Nr<  FrÇ  r?  z $A z $B rô   )rc   r1  rÅ   r2  r3  r   r   ré   r   rC  r[   r  r	   r	  rÂ   r  r  r  r  r   r  r  )	r“   rf  rl   r€   r  r  r  r  r  s	            rX   râ   ÚMarkupLMConverter.converted{  s(  € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%ØØ×1Ñ1×;Ñ;ñó

ˆ	ô #1×":Ò":ÈB×L_ÑL_Ñ"`ˆ	ÔÜ$×.Ò.Ó0ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜$˜s˜eÐ$Ø�5˜˜S˜E  c UÐ+àÐ#ØÐ#ðñ$
ˆ	Ô ð ÐrZ   rj   Nr  rj   rZ   rX   r	  r	  z  s   † ð"˜9÷ "rZ   r	  c                   ó0   • \ rS rSrSrS rS rS rS rSr	g)	ÚMoshiConverteri   Tc                 ó  • [        U S5        [        R                  X5        [        5       nUR	                  5       n[        US5       nUR                  UR                  5       5        S S S 5        X@l        g ! , (       d  f       N= f©NrQ   r�   ©	r   rÚ   r—   rY   rŽ   r�   r�   r‘   r’   ©r“   rŒ  r´   r”   r•   r–   s         rX   r—   ÚMoshiConverter.__init__£  se   € Ü˜$ 
Ô+ä×Ñ˜4Ô,ô $Ó%ˆ	à× Ñ Ó"ˆÜ�*˜dÔ# qØ×Ñ˜aŸf™f›hÔ'÷ $à�
÷ $Õ#úó   Á A7Á7
Bc                 óô   • UR                   R                  n[        R                  " SS5      /nU(       d  [        R                  " U5      $ [        R                  " [        R
                  " U5      /U-   5      $ rÝ  )r³   r§   r
   r  r]  r�  rª  s       rX   r   ÚMoshiConverter.normalizer°  sg   € Ø$×4Ñ4×IÑIÐä×Ò  UÓ+ð
ˆö $Ü×'Ò'¨Ó5Ð5ä×'Ò'¬×)@Ò)@ÐAUÓ)VÐ(WÐZfÑ(fÓgÐgrZ   c                 óî   • [         R                  " SS5      [         R                  " 5       [         R                  " 5       /nU(       a  U[         R                  " SSS9/-  n[         R
                  " U5      $ rö  rø  rù  s       rX   r	  ÚMoshiConverter.decoderº  rü  rZ   c                 ó0   • Sn[         R                  " XSS9$ )Nr`   Fr  )r   r°  r±  s       rX   r  ÚMoshiConverter.pre_tokenizerÄ  s   € Ø ˆÜ×'Ò'°KÐfkÑlÐlrZ   r½   N)
r¯   r¾   r¿   rÀ   r�  r—   r   r	  r  rÆ   rj   rZ   rX   r  r     s   † ØÐòòhò+õmrZ   r  c                   óL   • \ rS rSrSrSS jrS rS rS rS r	S	 r
S
 rS rSrg)ÚHeliumConverteriÉ  TNc                 ó  • [        U S5        [        R                  X5        [        5       nUR	                  5       n[        US5       nUR                  UR                  5       5        S S S 5        X@l        g ! , (       d  f       N= fr  r  r  s         rX   r—   ÚHeliumConverter.__init__Ì  sc   € Ü˜$ 
Ô+ä×Ñ˜4Ô,ä#Ó%ˆ	à× Ñ Ó"ˆÜ�*˜dÔ# qØ×Ñ˜aŸf™f›hÔ'÷ $à�
÷ $Õ#úr  c                 óR  • U R                  U5      n[        [        UU R                  U5      U R                  S95      n[        UR                  5       VVs/ s HR  u  pEUR                  S;   d  M  XER                  UR                  S:H  =(       d    UR                  U R                  ;   4PMT     nnnUR                  [        US S9 VVVs/ s H  u  pGn[        USUSS9PM     snnn5        UR                  [        S	SSS
9/5        UR                  SSS9  U$ s  snnf s  snnnf )Nr›  rœ   r�   c                 ó   • U S   $ r    rj   r¡   s    rX   rm   Ú+HeliumConverter.tokenizer.<locals>.<lambda>í  rŸ  rZ   ro   FT)r¤   r¥   Úsingle_wordÚ
r£   rë  )rú  Úpad_id)rl   r   r   r›   r�  r±   r¬   r²   r­   r÷   r¢  r}   r   Úenable_padding)	r“   r’   r   r  r·   r¸   r¹   rº   r¥   s	            rX   r  ÚHeliumConverter.tokenizerØ  s   € Ø—z‘z %Ó(ˆÜÜØØ—{‘{ 5Ó)Ø"×7Ñ7ñó
ˆ	ô # 5§<¡<Ô0ô
â0‘�Ø�v‰v˜Ñó IˆR—‘˜!Ÿ&™& A™+×G¨¯©°D×4GÑ4GÑ)GÓHÙ0ð 	ñ 
ð
 	×Ñô +1Ð1AÁ~Ò*Võâ*VÑ&�B˜wô ˜5¨U¸GÐQUÔVÙ*Vóô	
ð 	×Ñœj¨¸%ÈÑOÐPÔQØ× Ñ ¨7¸1Ð Ñ=ØÐùó
ùôs   ÁDÁ1?DÃD"c                 ó®   • / nUR                    HB  nUR                  S:X  a  USUR                  4/-  nM'  X#R                  UR                  4/-  nMD     U$ )Nz<0x0A>r"  r”  rò  s       rX   rl   ÚHeliumConverter.vocabô  sV   € ØˆØ—\”\ˆEØ�{‰{˜hÓ&Ø˜4 §¡Ð-Ð.Ñ.’àŸ;™;¨¯©Ð4Ð5Ñ5’ñ	 "ð
 ˆrZ   c                 ó
   • SnU$ r    rj   rÜ  s      rX   r›   ÚHeliumConverter.unk_idý  rÞ  rZ   c                 óà   • [         R                  " SS5      [         R                  " 5       [         R                  " 5       /nU[         R                  " SSS9/-  n[         R
                  " U5      $ rö  rø  rù  s       rX   r	  ÚHeliumConverter.decoder  sY   € ä×Ò˜U CÓ(Ü×!Ò!Ó#Ü�MŠM‹Oð
ˆð
 	”X—^’^¨C°aÑ8Ð9Ñ9ˆÜ× Ò  Ó*Ð*rZ   c                 ó„   • [         R                  " [         R                  " S5      [         R                  " SS5      /5      $ rÝ  )r
   r]  r   r  r˜  s     rX   r   ÚHeliumConverter.normalizer
  s2   € Ü×#Ò#¤[×%8Ò%8¸Ó%=¼{×?RÒ?RÐSWÐY^Ó?_Ð$`ÓaÐarZ   c                 óZ   • [         R                  " [         R                  " SS5      /5      $ )Nr"  Ú
contiguous)r   r]  r^  ré  s      rX   r  ÚHeliumConverter.pre_tokenizer  s#   € Ü×&Ò&¬×(<Ò(<¸TÀ<Ó(PÐ'QÓRÐRrZ   c                 ó8   • [         R                  " SS// SQS/S9$ )Nrá  r™  )rá  r™  rá  rš  )rá  r   rô   )r   r  rá   s    rX   r  ÚHeliumConverter.post_processor  s/   € Ü×,Ò,àØðòð ðñ
ð 	
rZ   r½   rÒ   )r¯   r¾   r¿   rÀ   r�  r—   r  rl   r›   r	  r   r  r  rÆ   rj   rZ   rX   r  r  É  s2   † ØÐô
òò8òò+òbòSõ
rZ   r  c                   ó(   • \ rS rSrSrSS jrS rSrg)ÚParakeetConverteri"  TNc                 ó  • Xl         [        U S5        [        R                  X5        [	        5       nUR                  5       n[        US5       nUR                  UR                  5       5        S S S 5        X@l	        g ! , (       d  f       N= fr  )
rŒ  r   rÚ   r—   rY   rŽ   r�   r�   r‘   r’   )r“   rŒ  r�  r”   r•   r–   s         rX   r—   ÚParakeetConverter.__init__%  sh   € Ø$Œä˜$ 
Ô+ä×Ñ˜4Ô,ä#Ó%ˆ	Ø× Ñ Ó"ˆÜ�*˜dÔ# qØ×Ñ˜aŸf™f›hÔ'÷ $à�
÷ $Õ#ús   Á A=Á=
Bc                 ó¸  • U R                  U5      nU R                  U R                  5      R                  U5      u  p4[	        U5       VVVs0 s H
  u  nu  pgXe_M     nnnn[        [        UUUR                  R                  SU R                  S S95      n	[	        UR                  5       V
Vs/ s HR  u  p«UR                  S;   d  M  X«R                  UR                  S:H  =(       d    UR                  U R                  ;   4PMT     nn
nU	R                  [        US S9 V
VVs/ s H  u  p­n[!        USUS9PM     snnn
5        U	$ s  snnnf s  snn
f s  snnn
f )	NTrœ  rœ   r�   c                 ó   • U S   $ r    rj   r¡   s    rX   rm   Ú-ParakeetConverter.tokenizer.<locals>.<lambda>K  rŸ  rZ   ro   Fr£   )rl   r   rŒ  r»   r±   r   r   r©   r}  r�  r¬   r²   r­   r÷   r¢  r}   r   )r“   r’   r   r£  r€   rµ   r¶   r®   r¤  r  r·   r¸   r¹   rº   r¥   s                  rX   r  ÚParakeetConverter.tokenizer2  sG  € Ø—z‘z %Ó(ˆà×%Ñ% d§o¡oÓ6×>Ñ>¸|ÓL‰	ˆÜ5>¸|Ô5LÕMÒ5LÑ!1 ¡M T�T’WÑ5Lˆ	ÒMÜÜØØØ×,Ñ,×6Ñ6ØØ"×7Ñ7Øñó	
ˆ	ô # 5§<¡<Ô0ô
â0‘�Ø�v‰v˜Ñó IˆR—‘˜!Ÿ&™& A™+×G¨¯©°D×4GÑ4GÑ)GÓHÙ0ð 	ñ 
ð
 	×Ñô +1Ð1AÁ~Ò*Võâ*VÑ&�B˜wô ˜5¨U¸GÔDÙ*Vóô	
ð Ðùô3 Nùó
ùôs   ÁEÂ/EÃ	?EÄ%E)r’   rŒ  rÒ   )r¯   r¾   r¿   rÀ   r�  r—   r  rÆ   rj   rZ   rX   r4  r4  "  s   † ØÐôõrZ   r4  c            	      óþ  • [        [        [        S5      [        S5      S-   5      5      [        [        [        S5      [        S5      S-   5      5      -   [        [        [        S5      [        S5      S-   5      5      -   n U SS nS	n[        S
5       H4  nX0;  d  M
  U R                  U5        UR                  S
U-   5        US-  nM6     U Vs/ s H  n[	        U5      PM     nn[        [        X5      5      $ s  snf )a  
Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
characters the bpe code barfs on.

The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
tables between utf-8 bytes and unicode strings.
Ú!Ú~r   ô   Â¡ô   Â¬ô   Â®ô   Ã¿Nr   é   )rÅ   r{   Úordr|   Úchrry   r»  )ÚbsÚcsÚnÚbs       rX   Úbytes_to_unicoderI  R  sÞ   € ô 	ŒU”3�s“8œS ›X¨™\Ó*Ó+¬d´5¼¸T»ÄCÈÃIÐPQÁMÓ3RÓ.SÑSÔVZÔ[`ÔadÐeiÓajÔloÐptÓluÐxyÑlyÓ[zÓV{Ñ{ð ð 
‰Aˆ€BØ	€AÜ�4Ž[ˆØ�;Ø�I‰I�aŒLØ�I‰I�d˜Q‘hÔØ�‰FŠAñ	 ñ
 Ó	š"�QŒ#ˆaŽ&™"€BÐ	Ü”�B“ÓÐùò 
s   ÃC:c                   óL   • \ rS rSrSr    SS jrS\4S jrS rS\	4S	 jr
S
rg)ÚTikTokenConverterij  z
A general tiktoken converter.
Nc                 óŒ   • Xl         X l        X0l        [        U[        5      (       a  UR                  5       U l        g UU l        g rÒ   )rŒ  rÿ  r[   r~  ry   r3  Úextra_special_tokens)r“   rŒ  rÿ  r[   rM  r´   s         rX   r—   ÚTikTokenConverter.__init__o  sA   € ð %ŒØŒØ 0Ôä+5Ð6JÌD×+QÑ+QÐ ×%Ñ%Ó'ð 	Õ!ØWkð 	Õ!rZ   Útiktoken_urlc                 ó>  ^^•  SSK Jn  U" U5      m[	        5       mU4S jn/ n0 nTR                  5        H“  u  pgXuU" U5      '   [        U5      S:X  a  M   / n[        S[        U5      5       H8  n	US U	 XiS  pºU
T;   d  M  UT;   d  M  X«-   T;   d  M%  UR                  X«U45        M:     [        UU4S jSS9nUR                  U5        M•     [        US	 SS9nU Vs/ s H  oÃ" US   5      U" US   5      4PM     nnXT4$ ! [         a    [        S5      ef = fs  snf )
Nr   )Úload_tiktoken_bpezY`tiktoken` is required to read a `tiktoken` file. Install it with `pip install tiktoken`.c           	      óˆ   >• SR                  U R                  S5       Vs/ s H  nT[        U5         PM     sn5      $ s  snf ©Nr<  zlatin-1©rº  ÚdecoderC  ©rH  ÚcharÚbyte_encoders     €rX   Útoken_bytes_to_stringÚPTikTokenConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_string‰  s8   ø€ Ø—7‘7ÀÇÁÈÔ@SÓTÒ@S¸˜L¬¨T«Ô3Ñ@SÑTÓUÐUùÒTó    ?r   c                 ó$   >• TU S      TU S      4$ ri   rj   )rk   r2  s    €rX   rm   ÚCTikTokenConverter.extract_vocab_merges_from_model.<locals>.<lambda>—  s   ø€ °¸1¸Q¹4±À)ÈAÈaÉDÁ/Ñ0RrZ   Frv   c                 ó   • U S   $ ©Nrr   rj   rt   s    rX   rm   r]  ™  ó   € °°A²rZ   )Útiktoken.loadrQ  r¡  Ú
ValueErrorrI  rz   rs   r{   r|   r}   r~   )r“   rO  rQ  rY  r€   rl   rº   Úrankrƒ   r„   r…   r†   ru   r2  rX  s                @@rX   Úextract_vocab_merges_from_modelÚ1TikTokenConverter.extract_vocab_merges_from_model~  sC  ù€ ð	Ý7ñ & lÓ3ˆ	Ü'Ó)ˆõ	Vð ˆØˆØ$Ÿ?™?Ö,‰KˆEØ26Ñ'¨Ó.Ñ/Ü�5‹z˜Q‹ÙØˆEÜ˜q¤# e£*Ö-�Ø#(¨¨% =°%¸°-˜Ø˜iÕ'¨G°yÕ,@ÀgÑFWÐ\eÕEeØ—L‘L '°DÐ!9Ö:ñ .ô ˜5Ô&RÐ\aÑbˆEØ�M‰M˜%Ö ñ -ô ˜Ñ$6ÀÑFˆÙ\bÓcÒ\bÐUXÐ(¨¨Q©Ó0Ñ2GÈÈAÉÓ2OÓPÑ\bˆÐcØˆ}Ðøô5 ó 	ÜØkóð ð	üò2 ds   „D ÃDÄDc                 ó¾   • U R                  U R                  5      u  p[        [        XSS95      n[	        UR
                  S5      (       a  SUR
                  l        U$ ©NF)r.  Úignore_mergesT©rd  rŒ  r   r   rü   r‹   rh  ©r“   r   r€   r  s       rX   r  ÚTikTokenConverter.tokenizer�  óM   € Ø#×CÑCÀDÇOÁOÓTÑˆÜœc ,ÀÑGÓHˆ	Ü�9—?‘? O×4Ñ4Ø,0ˆI�O‰OÔ)ØÐrZ   r\   c                 óÜ  • U R                  5       n[        R                  " [        R                  " [	        U R
                  5      SSS9[        R                  " U R                  SS9/5      Ul        [        R                  " 5       Ul
        U R                  b5  UR                  U R                   Vs/ s H  n[        USSS9PM     sn5        [        R                  " SS9Ul        U$ s  snf )NrV  FrW  rZ  Tr£   rA  )r  r   r]  r^  r   rÿ  rC  r[   r  r	   r	  rM  r5  r   r   r  )r“   r  rº   s      rX   râ   ÚTikTokenConverter.converted¤  sÊ   € Ø—N‘NÓ$ˆ	Ü"0×"9Ò"9ä×$Ò$¤U¨4¯<©<Ó%8À:ÐV[Ñ\Ü×(Ò(¸$×:OÑ:OÐ[`Ñaðó#
ˆ	Ôô %×.Ò.Ó0ˆ	Ôà×$Ñ$Ñ0Ø×(Ñ(ØPT×PiÒPiÓjÒPiÀu”˜E¨e¸TÔBÑPiÑjôô $.×#7Ò#7ÀUÑ#Kˆ	Ô àÐùò ks   Â2C))r[   rM  rÿ  rŒ  ©Nzs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+FN)r¯   r¾   r¿   rÀ   rÁ   r—   rÂ   rd  r  r   râ   rÆ   rj   rZ   rX   rK  rK  j  s:   † ñð ð KØØ!ô
ð¸Cô ò>ð˜9÷ rZ   rK  c                   óH   • \ rS rSr    S
S jrS\4S jrS rS\4S jr	S	r
g)ÚMistralConverteri¸  Nc                 óŒ   • Xl         X l        X0l        [        U[        5      (       a  UR                  5       U l        g UU l        g rÒ   )rŒ  rÿ  r[   r~  ry   r3  r¦   )r“   rŒ  rÿ  r[   r¦   r´   s         rX   r—   ÚMistralConverter.__init__¹  sG   € ð %ŒØŒØ 0Ôô Ð3´T×:Ñ:ð &×*Ñ*Ó,ð 	Õ&ð +ð 	Õ&rZ   rO  c                 ó,  ^^• SS K nSS Kn[        U R                  SSS9 nUR	                  U5      nS S S 5        WS   S   U l        US    Vs/ s H  n[        US   US	   S
9PM     snU l        US   n[        5       m[        U4S j5       n/ n	0 n
[        U R                  5       H  u  p¼XºUR                  '   M     U Vs/ s H  obR                  US   5      PM     nn[        U5      n[        U5       VVs0 s H  u  pìXÎ_M	     snnm[        [        USS95       H—  u  pìXêU" U5      '   [        U5      S:X  a  M   / n[!        S[        U5      5       H<  nUS U UUS  nnUU;   d  M  UU;   d  M  UU-   U;   d  M(  UR#                  UUU45        M>     [%        UU4S jSS9nU	R'                  U5        M™     [%        U	S SS9n	U	 Vs/ s H  nU" US   5      U" US   5      4PM     n	nX©4$ ! , (       d  f       GNÄ= fs  snf s  snf s  snnf s  snf )Nr   Úrzutf-8)ÚencodingÚconfigrÿ  r÷   Ú	token_strÚ
is_control)r¥   rl   c           	      óˆ   >• SR                  U R                  S5       Vs/ s H  nT[        U5         PM     sn5      $ s  snf rS  rT  rV  s     €rX   rY  ÚOMistralConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_string×  s8   ø€ à—7‘7ÀÇÁÈÔ@SÓTÒ@S¸˜L¬¨T«Ô3Ñ@SÑTÓUÐUùÒTr[  Útoken_bytesz(Converting tekken.json to tokenizer.json)Údescr   c                 ó$   >• TU S      TU S      4$ ri   rj   )rk   Útoken_to_ranks    €rX   rm   ÚBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>ë  s   ø€ °¸qÀ¹tÑ1DÀmÐTUÐVWÑTXÑFYÑ0ZrZ   Frv   c                 ó   • U S   $ r_  rj   rt   s    rX   rm   r€  í  r`  rZ   )Úbase64Újsonr�   rŒ  Úloadrÿ  r   r¦   rI  r   r±   r÷  Ú	b64decoderx   r   rs   r{   r|   r}   r~   )r“   rO  r‚  rƒ  r–   ÚuntypedÚkr2  rY  r€   rl   Úidxrº   Úrank_setrc  rƒ   r„   r…   r†   ru   rX  r  s                       @@rX   rd  Ú0MistralConverter.extract_vocab_merges_from_modelÊ  s*  ù€ ÛÛä�$—/‘/ 3°Ò9¸QØ—i‘i “lˆG÷ :à˜xÑ(¨Ñ3ˆŒàIPÐQaÒIbó*
ÚIbÀAŒJ�q˜‘~¨q°©Ô?ÑIbñ*
ˆÔ&ð ˜GÑ$ˆ	Ü'Ó)ˆä	ô	Vó 
ð	Vð ˆØˆÜ# D×$BÑ$BÖC‰JˆCØ#&�%—-‘-Ó ñ DáAJÓKÂ¸A×%Ñ% a¨Ñ&6Ö7Áˆ	ÐKÜ�y“>ˆÜ8AÀ)Ô8LÔMÒ8L©¨˜šÑ8LÒMˆÜ$¤T¨)Ð:dÑ%eÖf‰KˆDØ26Ñ'¨Ó.Ñ/Ü�5‹z˜Q‹ÙØˆEÜ˜q¤# e£*Ö-�Ø#(¨¨% =°%¸¸°-˜�Ø˜hÕ&¨7°hÕ+>ÀGÈgÑDUÐZbÕCbØ—L‘L '¨7°DÐ!9Ö:ñ .ô ˜5Ô&ZÐdiÑjˆEØ�M‰M˜%Ö ñ gô ˜Ñ$6ÀÑFˆÙ\bÓcÒ\bÐUXÑ(¨¨Q©Ó0Ñ2GÈÈAÉÓ2OÓPÑ\bˆÐcØˆ}Ð÷C :Ö9üò*
ùò LùãMùò ds#    G/ÁHÃHÃ:HÇ
 HÇ/
G>c                 ó¾   • U R                  U R                  5      u  p[        [        XSS95      n[	        UR
                  S5      (       a  SUR
                  l        U$ rg  ri  rj  s       rX   r  ÚMistralConverter.tokenizerñ  rl  rZ   r\   c                 ó„  • U R                  5       n[        R                  " [        R                  " [	        U R
                  5      SSS9[        R                  " U R                  SS9/5      Ul        [        R                  " 5       Ul
        UR                  U R                  5        [        R                  " SS9Ul        U$ )NrV  FrW  rZ  rA  )r  r   r]  r^  r   rÿ  rC  r[   r  r	   r	  r¢  r¦   r   r  )r“   r  s     rX   râ   ÚMistralConverter.convertedø  s™   € Ø—N‘NÓ$ˆ	Ü"0×"9Ò"9ä×$Ò$¤U¨4¯<©<Ó%8À:ÐV[Ñ\Ü×(Ò(¸$×:OÑ:OÐ[`Ñaðó#
ˆ	Ôô %×.Ò.Ó0ˆ	Ôà×Ñ˜T×;Ñ;Ô<Ü#-×#7Ò#7ÀUÑ#Kˆ	Ô àÐrZ   )r[   r¦   rÿ  rŒ  ro  )r¯   r¾   r¿   rÀ   r—   rÂ   rd  r  r   râ   rÆ   rj   rZ   rX   rq  rq  ¸  s6   † ð ð KØØ"&ô
ð"%¸Cô %òNð˜9÷ rZ   rq  ÚAlbertTokenizerÚBartTokenizerÚBarthezTokenizerÚBertTokenizerÚBigBirdTokenizerÚBlenderbotTokenizerÚCamembertTokenizerÚCLIPTokenizerÚCodeGenTokenizerÚConvBertTokenizerÚDebertaTokenizerÚDebertaV2TokenizerÚDistilBertTokenizerÚDPRReaderTokenizerÚDPRQuestionEncoderTokenizerÚDPRContextEncoderTokenizerÚElectraTokenizerÚFNetTokenizerÚFunnelTokenizerÚGPT2TokenizerÚHerbertTokenizerÚLayoutLMTokenizerÚLayoutLMv2TokenizerÚLayoutLMv3TokenizerÚLayoutXLMTokenizerÚLongformerTokenizerÚLEDTokenizerÚLxmertTokenizerÚMarkupLMTokenizerÚMBartTokenizerÚMBart50TokenizerÚMPNetTokenizerÚMobileBertTokenizerÚMvpTokenizerÚNllbTokenizerÚOpenAIGPTTokenizerÚPegasusTokenizerÚQwen2TokenizerÚReformerTokenizerÚRemBertTokenizerÚRobertaTokenizerÚRoFormerTokenizerÚSeamlessM4TTokenizerÚSqueezeBertTokenizerÚT5TokenizerÚUdopTokenizerÚWhisperTokenizerÚXLMRobertaTokenizerÚXLNetTokenizerÚSplinterTokenizerÚXGLMTokenizer)ÚLlamaTokenizerÚCodeLlamaTokenizerÚGemmaTokenizerÚPhi3Tokenizerc                 ó<  • U R                   R                  nU[        ;   a&  U(       d  [        U   nU" U 5      R                  5       $ U R                  R                  S5      (       a>  X l        [        R                  S5        [        U R                  5      R                  5       $  [        R                  S5        [        U R                  U R                  S9R                  5       $ ! [         a*    [        S[        [        R                  5       5       35      ef = f)a\  
Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

Args:
    transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
        Instance of a slow tokenizer to convert in the backend tokenizer for
        [`~tokenization_utils_base.PreTrainedTokenizerFast`].
   from_tiktoken (bool, optional): Whether to use the `tiktoken` library to convert the tokenizer instead of sentencepiece.
        Defaults to False.

Return:
    A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
    [`~tokenization_utils_base.PreTrainedTokenizerFast`]
ztekken.jsonz#Converting from Mistral tekken.jsonzConverting from Tiktoken)rŒ  rM  zÌConverting from SentencePiece and Tiktoken failed, if a converter for SentencePiece is available, provide a model path with a SentencePiece tokenizer.model file.Currently available slow->fast converters: )r‘  r¯   ÚSLOW_TO_FAST_CONVERTERSrâ   rŒ  Úendswithrc   ÚloggerÚinforq  rK  rM  r¡  rb  rÅ   r3  )Útransformer_tokenizerÚfrom_tiktokenÚtokenizer_class_nameÚconverter_classs       rX   Úconvert_slow_tokenizerrÏ  C  sþ   € ð  1×:Ñ:×CÑCÐØÔ6Ó6¾}Ü1Ð2FÑGˆÙÐ4Ó5×?Ñ?ÓAÐAØ	×	)Ñ	)×	2Ñ	2°=×	AÑ	AØ3HÔ0Ü�‰Ð9Ô:ÜÐ 5× @Ñ @ÓA×KÑKÓMÐMð	Ü�K‰KÐ2Ô3Ü$Ø0×;Ñ;Ø%:×%OÑ%Oñ÷ ‰i‹kðøô ó 	Üð>ä>BÔCZ×C_ÑC_ÓCaÓ>bÐ=cðeóð ð	ús   Â&A C' Ã'4D)r<  rÒ   )F)WrÁ   rŽ  Úcollections.abcr   Ú	functoolsr   Ú	packagingr   Ú
tokenizersr   r   r   r	   r
   r   r   rª   r   r   r   r   Úutilsr   r   r   r   Úutils.import_utilsr   Ú
get_loggerr¯   rÉ  r=  rK  rY   ÚboolrÂ   re   r‡   r‰   rÈ   rØ   rÚ   rå   r  r!  r%  r)  r:  rK  rS  ra  rj  rr  rz  r½  rÙ  ræ  r  r  rA  rN  r^  ri  rs  r{  r  r…  r‡  rŸ  r±  r¶  rÁ  rÅ  rË  rÏ  rÓ  rÛ  rð  r	  r  r  r4  rI  rK  rq  rÇ  rÏ  rj   rZ   rX   Ú<module>rØ     s³  ðñó Ý &Ý å ß f× fÑ fß 5Ñ 5Ý ç `Ó `Ý 5ð 
×	Ò	˜HÓ	%€ò€ð8 $ò 'ñ Ð ô>Gð"¨$ð Àsô ñ°jÀ±oÈÑ6Lõ ÷02ñ 2ôjÐ"8ô ð"I˜cð I dô I÷$ñ $ô$�Iô $ôN/˜	ô /ôd$�iô $ôN$�Yô $ôN˜ô ô6#�Iô #ôL�yô ô>)�Yô )ôX�yô ô:$˜	ô $ôN�yô ô>x�9ô xôv"
�lô "
ôJ
�|ô 
ô -˜ô -ô`
˜ô 
ôBG�\ô GôT-�|ô -ô`,�Lô ,ô^
˜<ô 
ô2,˜,ô ,ô^"
�\ô "
ôJ	˜ô 	ô
�|ô 
ô@	˜lô 	ô;x�|ô ;xô|�,ô ô8
�Lô 
ô!�yô !ôH	
�|ô 	
ô(�Iô (ôV$˜)ô $ôN˜)ô ô:
�Lô 
ô61
�\ô 1
ôh+�\ô +ô\#˜	ô #ôL&m�\ô &môRV
�lô V
ôr-˜ô -ò`÷0Kñ K÷\Mñ Mð`8Ø�ð8àÐ%ð8ð Ð(ð8ð �]ð	8ð
 Ð(ð8ð Ð.ð8ð Ð,ð8ð �]ð8ð ˜ð8ð ˜ð8ð Ð(ð8ð Ð,ð8ð ˜=ð8ð ˜-ð8ð " =ð8ð  ! -ð!8ð" ˜ñ#8ð$ �_ð%8ð& �ð'8ð( �]ð)8ð* Ð(ð+8ð, ˜ð-8ð. ˜=ð/8ð0 Ð+ð18ð2 Ð-ð38ð4 Ð+ð58ð6 Ð$ð78ð8 �}ð98ð: Ð*ð;8ð< �nð=8ð> Ð(ð?8ð@ �nðA8ðB ˜=ðC8ðD Ð$òE8ðF �]ðG8ðH Ð,ðI8ðJ Ð(ðK8ðL �nðM8ðN Ð*ðO8ðP Ð(ðQ8ðR Ð(ðS8ðT Ð*ðU8ðV Ð0ðW8ðX ˜MðY8ðZ �;ð[8ð\ �]ð]8ð^ Ð(ð_8ð` Ð.ða8ðb �nðc8ðd Ð*ðe8ðf �]ñg8ðh %Ø(Ø$Ø#òo8Ð ñv$È)ö $rZ   