ó
    pyüix  ã                   óÚ  • S r SSKJr  SSKrSSKJrJrJrJrJ	r	  SSK
JrJr  SSKJr  SSKJrJrJrJrJr  SS	KJr  SS
KJr  \R2                  " \5      r0 SSSS._SSSSSSSSSSSS.
_SSSSSSSSSSSS.
_SSSSSSSSSSSS.
_SSSSSSSSSSSSSS ._S!SSSSSSSSSSS"SS#S$._S%SSSSSSSSSSS&S'._S(SSSSSSSSSSS.
_S)SSSSSSSSSSSSSS*._S+SSSSSSSSSSS.
_S,S-S.S/S0S1._S2SSSSSSSSSSS.
_S3S4SS5SS6S7._S8S9S:S;S<S=S>SS6S?S@SSA._SBSSSSSSSSCSSD.	_SES4SFSGSHS5S6SI._SJSSSSSSSKSL._SSSS6SSMSNSOSSP.	SSSSSSSSSQSS.
SSSSSSSSSSS#SSR.SSSSSSSSSSS#SSR.S9S:S;S<S=S>SS6S?S@SSA.SSSSSSSSSSS.
0 SSS_STS_SHS_SUS_SVSW_SXS_SYS_SZS_S[S_S\S_S]S_S^S"_S_S_S`S_SS_SaSb_Sc.ErSdSeSfSgShS-S.S/S0SiSj.
SkSS-S.S/S0Sl.Sm.rSnSo0SpSo0Sq.rSr r " Ss St5      r  " Su Sv\5      r! " Sw Sx\5      r" " Sy Sz\5      r# " S{ S|\5      r$ " S} S~\5      r% " S S€\5      r&0 S\!_S\"_S\"_S(\"_S)\"_S2\#_S3\$_S+\$_SB\$_SE\$_SJ\$_S8\%_S�\$_S‚\$_Sƒ\&_S„\&_S…\%_\!\!\"S†.Er'S‡\(Sˆ\)\\*4   4S‰ jr+g)Šz
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
é    )ÚarrayN)Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramé   )Ú
AddedToken)ÚGemmaConverterÚGPT2ConverterÚLlamaConverterÚQwen2ConverterÚT5Converter)Úlogging)ÚtqdmÚgeneralÚ
model_typeÚ_model_name_or_path)ÚarchitectureÚnameÚllamaÚmax_position_embeddingsÚnum_hidden_layersÚintermediate_sizeÚhidden_sizeÚhead_dimÚ
rope_thetaÚnum_attention_headsÚnum_key_value_headsÚrms_norm_epsÚ
vocab_size)
Úcontext_lengthÚblock_countÚfeed_forward_lengthÚembedding_lengthúrope.dimension_countúrope.freq_baseúattention.head_countúattention.head_count_kvú attention.layer_norm_rms_epsilonr#   ÚmistralÚqwen2Ú	qwen2_moeÚnum_expertsÚnum_experts_per_tok)r$   r%   r&   r'   r(   r)   r*   r+   r,   r#   Úexpert_countÚexpert_used_countÚgpt_ossÚnum_local_expertsÚsliding_window)r$   r%   r&   r'   r(   r)   r*   r+   r,   r#   r2   r3   r6   Úlfm2Úconv_L_cache)r$   r%   r&   r'   r(   r)   r*   r+   r,   r#   zshortconv.l_cacheÚqwen3Ú	qwen3_moe)r$   r%   r&   r'   r(   r)   úattention.key_lengthr*   r+   r,   r#   r2   r3   ÚfalconÚ	tokenizerÚbos_token_idÚeos_token_idÚunk_token_idÚpad_token_id)úggml.bos_token_idúggml.eos_token_idúggml.unknown_token_idúggml.padding_token_idÚphi3ÚbloomÚn_layerÚn_headÚlayer_norm_epsilon)r%   r'   r*   r#   úattention.layer_norm_epsilonÚt5Ún_positionsÚ
num_layersÚd_ffÚd_modelÚd_kvÚ	num_headsÚrelative_attention_num_bucketsÚdecoder_start_token_id)r$   r%   r&   r'   r;   r*   r+   rK   z attention.relative_buckets_countrT   r#   ÚstablelmÚlayer_norm_eps)	r$   r%   r&   r'   r(   r*   r+   rK   r#   Úgpt2Ún_ctxÚn_embdr&   )r%   r$   r'   r&   r*   rK   Ú
starcoder2Únorm_epsilon)r%   r$   r'   r&   r*   r+   rK   Úconv_kernelÚ
state_sizeÚtime_step_rank)	r#   r$   r'   r,   r%   zssm.conv_kernelzssm.state_sizezssm.time_step_rankzssm.inner_sizeÚnorm_eps)r$   r%   r&   r'   r(   r)   r;   r*   r+   r,   zattention.sliding_windowr#   r$   r%   r'   r(   Ú
rotary_dimr)   r*   r+   r;   zattention.value_lengthr,   r2   r3   Úexpert_feed_forward_lengthÚexpert_gating_funcÚscoring_func)ÚmambaÚnemotronÚgemma2Úgemma3Úumt5ÚdeciÚ
minimax_m2Útokenizer_typeÚtokensÚscoresÚ
token_typeÚmergesÚadd_prefix_space)
ú
ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesrB   rC   rD   rE   zggml.add_space_prefixÚchat_template)rr   rq   rB   rC   rD   rE   )r=   Útokenizer_configÚnorm_topk_probTÚuse_routing_bias)r:   rj   c                 ó¬  • [        U[        5      (       d  U/n[        U5      S:X  a  US   nS nOUS   S:w  a  [        S5      eUu  pUS;   a  [	        U S   5      n U $ US;   a  [        U S   5      n U $ US:X  a  [        U S   5      n U $ US:X  a3  [        S	[        U 5      5      R                  5       R                  5       n U $ US:X  a  [        X5      n U $ )
Né   r   é	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   rw   r   é   é   é   é
   é   )é   é   é   é   ÚB)Ú
isinstanceÚlistÚlenÚ
ValueErrorÚintÚfloatÚboolr   ÚtobytesÚdecodeÚ_gguf_parse_value)Ú_valueÚ	data_typeÚarray_data_types      Ú[/home/mande/repo/quber/.venv/lib/python3.13/site-packages/transformers/integrations/ggml.pyrŒ   rŒ   m  só   € Ü�i¤×&Ñ&Ø�Kˆ	Ü
ˆ9ƒ~˜ÓØ˜a‘Lˆ	Ø‰à�Q‰<˜1ÓÜÐoÓpÐpØ%.Ñ"ˆ	àÐ.Ó.Ü�V˜A‘Y“ˆð €Mð 
�gÓ	Ü�v˜a‘yÓ!ˆð €Mð 
�a‹Ü�f˜Q‘i“ˆð
 €Mð	 
�a‹Ü�sœD ›LÓ)×1Ñ1Ó3×:Ñ:Ó<ˆð €Mð 
�a‹Ü" 6Ó;ˆØ€Mó    c                   ó   • \ rS rSrS rSrg)ÚGGUFTokenizerSkeletoni…  c                 óÆ  ^• UR                  5        H  u  p#[        XU5        M     [        U S5      (       Gd4  [        U S5      (       a  [        U S5      (       d  [        S5      eU R                  nU R
                  n[        U5       VVs0 s H
  u  pgXuU   _M     snnm[        R                  S5        / n[        TR                  5       5       Hl  u  pš/ n[        S[        U	5      5       H,  nU	S U XœS  píXÔ;   d  M  Xä;   d  M  UR                  XÞU
45        M.     [        UU4S jSS	9nUR                  U5        Mn     [        US
 SS	9nU Vs/ s H  oÿS   US   4PM     nnX€l        OU R                   V	s/ s H  n	[!        U	R#                  S5      5      PM     sn	U l        [        U S5      (       d4  [        [        U R                  5      5       Vs/ s H  nS PM     snU l        [        U S5      (       d  / U l        [        U S5      (       d  S U l        [        U S5      (       a   U R&                  c  U R(                  U l        g g g s  snnf s  snf s  sn	f s  snf )Nro   rl   rm   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.z:Merges were not in checkpoint, building merges on the fly.rw   c                 ó$   >• TU S      TU S      4$ )Nr   rw   © )ÚxÚvocabs    €r�   Ú<lambda>Ú0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>›  s   ø€ °U¸1¸Q¹4±[À%ÈÈ!ÉÁ+Ñ4Nr‘   T)ÚkeyÚreversec                 ó   • U S   $ )Nr   r–   )Úvals    r�   r™   rš   �  s   € °C¸²Fr‘   r   Ú Úadded_tokensr@   Úunknown_token_id)ÚitemsÚsetattrÚhasattrr†   rl   rm   Ú	enumerateÚloggerÚwarningr   Úranger…   ÚappendÚsortedÚextendro   ÚtupleÚsplitr    r@   r¡   )ÚselfÚdict_ÚkÚvrl   rm   ÚiÚtro   ÚmergeÚpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rrž   Ú_r˜   s                    @r�   Ú__init__ÚGGUFTokenizerSkeleton.__init__†  s  ø€ Ø—K‘K–M‰DˆAÜ�D˜QÖñ "ô �t˜X×&Ò&Ü˜4 ×*Ñ*´'¸$À×2IÑ2IÜ Øróð ð —[‘[ˆFØ—[‘[ˆFÜ.7¸Ô.?Ô@Ò.?¡d a�Q˜q™	’\Ñ.?Ò@ˆEä�N‰NÐWÔXØˆFÜ&*¨5¯;©;«=Ö&9Ñ"�Ø�Ü" 1¤c¨%£jÖ1�EØ',¨V¨e }°e¸F°m˜WØÕ(¨WÕ->ØŸ™ g¸Ð%DÖEñ 2ô ˜uÔ*NÐX\Ñ]�Ø—‘˜eÖ$ñ ':ô ˜FÑ(:ÀDÑIˆFÙ28Ó9²&¨3˜1‘v˜s 1™vÓ&±&ˆFÐ9Ø �Kà@DÇÂÓLÂ°uœ5 §¡¨SÓ!1Ö2ÁÑLˆDŒKÜ˜4 ×*Ñ*Ü-2´3°t·{±{Ó3CÔ-DÓEÒ-D¨›tÑ-DÑE�”ä�t˜^×,Ñ,Ø "ˆDÔä�t˜^×,Ñ,Ø $ˆDÔô �4Ð+×,Ñ,°×1BÑ1BÑ1JØ $× 5Ñ 5ˆDÕð 2KÐ,ùó7 Aùò :ùò MùâEs   ÂIÅIÅ;$IÇI)r    ro   rm   r@   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r»   Ú__static_attributes__r–   r‘   r�   r“   r“   …  s   † õ'6r‘   r“   c                   ó8   • \ rS rSrS rS rS rS rS rS r	Sr
g	)
ÚGGUFLlamaConverteri°  c                 ó’   • [        U5      U l        U R                  U l        0 U l        [	        U R                  SS5      S:g  U l        g )Nrk   r   )r“   ÚprotoÚoriginal_tokenizerÚadditional_kwargsÚgetattrÚis_llama_3_tokenizer©r®   Útokenizer_dicts     r�   r»   ÚGGUFLlamaConverter.__init__±  s>   € Ü*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÔÜ$+¨D¯J©JÐ8HÈ'Ó$RÐV]Ñ$]ˆÕ!r‘   c                 óT   • [        [        UR                  UR                  5      5      $ ©N©r„   Úziprl   rm   ©r®   rÅ   s     r�   r˜   ÚGGUFLlamaConverter.vocab·  ó   € Ü”C˜Ÿ™ e§l¡lÓ3Ó4Ð4r‘   c                 ó   • UR                   $ rÎ   ©ro   rÑ   s     r�   ro   ÚGGUFLlamaConverter.mergesº  ó   € Ø�|‰|Ðr‘   c                 óÎ  • U R                  U R                  5      nU R                  U R                  5      n[        U5       VVVs0 s H
  u  nu  pVXT_M     nnnnUR                  b  UR
                  UR                     OS n[        USS 5      b  UR
                  UR                     OS n	[        USS 5      b  UR
                  UR                     OS n
[        [        UUUSSS95      n/ n[        U R                  S5      (       dX  Ub  UR                  [        USSS95        U	b  UR                  [        U	SSS95        U
b  UR                  [        U
SSS95        O~[        R                  " [        R                  " U R                  R                   5      S:H  5      S	   nU H4  nUR                  [        U R                  R
                  U   SSS95        M6     [#        U5      S	:w  a  UR%                  U5        [#        U R                  R&                  5      S	:w  a?  UR)                  U R                  R&                   Vs/ s H  n[        USSS9PM     sn5        X€R*                  S
'   X�R*                  S'   X R*                  S'   U R,                  (       a>  S U R*                  S'   SU R*                  S'   SU R*                  S'   SU R.                  l        U$ s  snnnf s  snf )Nr>   r?   T)Ú	unk_tokenÚfuse_unkÚbyte_fallbackrn   F©Ú
normalizedÚspecialry   r   rÙ   Ú	eos_tokenÚ	bos_tokenrp   Úclean_up_tokenization_spacesÚlegacy)r˜   rÅ   ro   r¥   r@   rl   rÈ   r>   r   r	   r¤   r©   r   ÚnpÚwherer   rn   r…   Úadd_special_tokensr    Ú
add_tokensrÇ   rÉ   rÆ   râ   )r®   rÅ   Úvocab_scoresro   r²   ÚwordÚ_scoreÚ	bpe_vocabrÙ   rà   rß   r=   Úspecial_tokensÚspecial_tokens_idxÚidxÚadded_tokens                   r�   r=   ÚGGUFLlamaConverter.tokenizer½  s¡  € Ø—z‘z $§*¡*Ó-ˆØ—‘˜TŸZ™ZÓ(ˆÜ6?ÀÔ6MÕNÒ6MÑ!2 ¡N T�T’WÑ6Mˆ	ÒNà8=×8JÑ8JÑ8V�E—L‘L ×!3Ñ!3Ò4Ð\`ˆ	Ü8?ÀÀ~ÐW[Ó8\Ñ8h�E—L‘L ×!3Ñ!3Ò4Ðnrˆ	Ü8?ÀÀ~ÐW[Ó8\Ñ8h�E—L‘L ×!3Ñ!3Ò4Ðnrˆ	äÜØØØ#ØØ"ñó
ˆ	ð ˆä�t—z‘z <×0Ñ0ØÑ$Ø×%Ñ%¤j°ÀuÐVZÑ&[Ô\àÑ$Ø×%Ñ%¤j°ÀuÐVZÑ&[Ô\àÑ$Ø×%Ñ%¤j°ÀuÐVZÑ&[Ô\øô "$§¢¬"¯(ª(°4·:±:×3HÑ3HÓ*IÈQÑ*NÓ!OÐPQÑ!RÐã)�Ø×%Ñ%¤j°·±×1BÑ1BÀ3Ñ1GÐTYÐcgÑ&hÖiñ *ô ˆ~Ó !Ó#Ø×(Ñ(¨Ô8äˆt�z‰z×&Ñ&Ó'¨1Ó,Ø× Ñ Ø]a×]gÑ]g×]tÒ]tÓuÒ]tÈk”˜K°EÀ5ÔIÑ]tÑuôð /8×Ñ˜{Ñ+Ø.7×Ñ˜{Ñ+Ø.7×Ñ˜{Ñ+à×$×$Ø9=ˆD×"Ñ"Ð#5Ñ6ØEIˆD×"Ñ"Ð#AÑBà/4ˆD×"Ñ" 8Ñ,Ø-2ˆD×#Ñ#Ô*àÐùôk OùòN vs   ÁKÉK"c                 óD  • [         R                  " 5       [         R                  " 5       [         R                  " SS5      /nU R                  (       a  U[         R
                  " SSSS9/-  nU(       a  U[         R                  " SSS9/-  n[         R                  " U5      $ )Nõ   â–�rŸ   FT©rp   Útrim_offsetsÚ	use_regexrw   ©ÚcontentÚleft)r   ÚByteFallbackÚFuseÚReplacerÉ   Ú	ByteLevelÚStripÚSequence©r®   Úreplacementrp   Úsequences       r�   ÚdecoderÚGGUFLlamaConverter.decoder÷  sƒ   € ä×!Ò!Ó#Ü�MŠM‹OÜ×Ò˜U CÓ(ð
ˆð ×$×$Øœ×+Ò+¸UÐQVÐbfÑgÐhÑhˆHæØœŸš°¸!Ñ<Ð=Ñ=ˆHÜ× Ò  Ó*Ð*r‘   c                 ó  • U R                  U R                  5      nU R                  U R                  5      nUb  X!l        SnSn[        U R                  S5      (       a  U R                  R
                  nU R                  X45      nUb  XQl        U R                  X45      Ul        U R                  5       nU(       a  Xal        U R                  (       a6  [        R                  " SSSS9Ul        [        R                  " / 5      Ul        U$ )Nrñ   Trp   Frò   )r=   rÅ   Ú
normalizerr¤   rÆ   rp   Úpre_tokenizerr  Úpost_processorrÉ   r   rû   r   rý   )r®   r=   r  rÿ   rp   r  r  s          r�   Ú	convertedÚGGUFLlamaConverter.converted  sé   € à—N‘N 4§:¡:Ó.ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÑ!Ø#-Ô àˆØÐÜ�4×*Ñ*Ð,>×?Ñ?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;ÓIˆØÑ$Ø&3Ô#à ŸL™L¨ÓGˆ	ÔØ×,Ñ,Ó.ˆÞØ'5Ô$ð ×$×$Ü&4×&>Ò&>Ø!&°UÀdñ'ˆIÔ#ô
 $/×#7Ò#7¸Ó#;ˆIÔ àÐr‘   )rÇ   rÉ   rÆ   rÅ   N)r½   r¾   r¿   rÀ   r»   r˜   ro   r=   r  r  rÁ   r–   r‘   r�   rÃ   rÃ   °  s"   † ò^ò5òò8òt+õ!r‘   rÃ   c                   ó6   ^ • \ rS rSrS rS\4U 4S jjrSrU =r$ )ÚGGUFQwen2Converteri)  c                 ó2   • [        U5      U l        0 U l        g rÎ   ©r“   rÆ   rÇ   rÊ   s     r�   r»   ÚGGUFQwen2Converter.__init__*  ó   € Ü"7¸Ó"GˆÔØ!#ˆÕr‘   Úreturnc           
      ó"  >• [        U R                  R                  5       VVs0 s H  u  pX!_M	     nnnU R                  R                  n[        TU ]  X45      nUR                  [        SSSS9[        SSSS9[        SSSS9/5        U$ s  snnf )Nú<|endoftext|>FTrÜ   z<|im_start|>z
<|im_end|>)r¥   rÆ   rl   ro   Úsuperr  rå   r   ©r®   r²   rè   r˜   ro   r=   Ú	__class__s         €r�   r  ÚGGUFQwen2Converter.converted.  s“   ø€ Ü(1°$×2IÑ2I×2PÑ2PÔ(QÔRÒ(Q™W˜Q�’Ñ(QˆÑRØ×(Ñ(×/Ñ/ˆÜ‘GÑ% eÓ4ˆ	à×$Ñ$ä˜?°uÀdÑKÜ˜>°eÀTÑJÜ˜<°EÀ4ÑHðô	
ð Ðùó Ss   ¤B©rÇ   rÆ   ©	r½   r¾   r¿   rÀ   r»   r   r  rÁ   Ú__classcell__©r  s   @r�   r
  r
  )  s   ø† ò$ð˜9÷ õ r‘   r
  c                   ó@   • \ rS rSrS rS rS rS rS rS\	4S jr
S	rg
)ÚGGUFPhi3Converteri=  c                 óT   • [        U5      U l        U R                  U l        0 U l        g rÎ   ©r“   rÅ   rÆ   rÇ   rÊ   s     r�   r»   ÚGGUFPhi3Converter.__init__>  s"   € Ü*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÕr‘   c                 óT   • [        [        UR                  UR                  5      5      $ rÎ   rÏ   rÑ   s     r�   r˜   ÚGGUFPhi3Converter.vocabC  rÓ   r‘   c                 ó   • UR                   $ rÎ   rÕ   rÑ   s     r�   ro   ÚGGUFPhi3Converter.mergesF  r×   r‘   c                 óÂ  • U R                  U R                  5      nU R                  U R                  5      n[        U5       VVVs0 s H
  u  nu  pVXT_M     nnnn[	        [        Xs5      5      nUR                  [        SSSSSS9[        SSSS9[        SSSSS9[        S	SSSS9[        S
SSSS9[        SSSSS9[        SSSSS9[        SSSSS9[        SSSSS9[        SSSSS9[        SSSSS9[        SSSSS9/5        UR                  b  UR                  UR                     OS U R                  S'   UR                  b  UR                  UR                     OS U R                  S'   UR                  b  UR                  UR                     OS U R                  S'   UR                  b  UR                  UR                     OS U R                  S'   U$ s  snnnf )Nú</s>TF)ÚrstripÚlstriprÝ   rÞ   r  rÜ   z<|assistant|>)r%  rÝ   rÞ   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>rÙ   rß   rà   Ú	pad_token)r˜   rÅ   ro   r¥   r   r	   rå   r   r@   rl   rÇ   r?   r>   rA   )	r®   rÅ   rç   ro   r²   rè   ré   rê   r=   s	            r�   r=   ÚGGUFPhi3Converter.tokenizerI  sú  € Ø—z‘z $§*¡*Ó-ˆØ—‘˜TŸZ™ZÓ(ˆÜ6?ÀÔ6MÕNÒ6MÑ!2 ¡N T�T’WÑ6Mˆ	ÒNäœc )Ó4Ó5ˆ	à×$Ñ$ä˜6¨$°uÈÐX\Ñ]Ü˜?°uÀdÑKÜ˜?°4ÀEÐSWÑXÜÐ-°dÀuÐVZÑ[ÜÐ-°dÀuÐVZÑ[ÜÐ-°dÀuÐVZÑ[ÜÐ-°dÀuÐVZÑ[Ü˜<°ÀÐPTÑUÜ˜9¨T¸eÈTÑRÜÐ-°dÀuÐVZÑ[ÜÐ-°dÀuÐVZÑ[Ü˜:¨d¸uÈdÑSðô	
ð$ 16×0BÑ0BÑ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÑ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÑ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÑ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð ÐùôE Os   ÁGc                 óî   • [         R                  " 5       [         R                  " 5       [         R                  " US5      /nU(       a  U[         R                  " SSS9/-  n[         R
                  " U5      $ )NrŸ   rw   rõ   )r   rø   rù   rú   rü   rý   rþ   s       r�   r  ÚGGUFPhi3Converter.decoderp  s\   € ä×!Ò!Ó#Ü�MŠM‹OÜ×Ò˜[¨#Ó.ð
ˆö ØœŸš°¸!Ñ<Ð=Ñ=ˆHÜ× Ò  Ó*Ð*r‘   r  c                 óÒ   • U R                  U R                  5      nSnSn[        U R                  S5      (       a  U R                  R                  nU R                  X#5      Ul        U$ )Nrñ   Trp   )r=   rÅ   r¤   rÆ   rp   r  )r®   r=   rÿ   rp   s       r�   r  ÚGGUFPhi3Converter.converted{  s\   € Ø—N‘N 4§:¡:Ó.ˆ	àˆØÐÜ�4×*Ñ*Ð,>×?Ñ?Ø#×6Ñ6×GÑGÐà ŸL™L¨ÓGˆ	ÔàÐr‘   ©rÇ   rÆ   rÅ   N)r½   r¾   r¿   rÀ   r»   r˜   ro   r=   r  r   r  rÁ   r–   r‘   r�   r  r  =  s(   † ò$ò
5òò%òN	+ð
˜9÷ 
r‘   r  c                   ó6   ^ • \ rS rSrS rS\4U 4S jjrSrU =r$ )ÚGGUFGPTConverteriˆ  c                 ó2   • [        U5      U l        0 U l        g rÎ   r  rÊ   s     r�   r»   ÚGGUFGPTConverter.__init__‰  r  r‘   r  c                 óÄ   >• [        U R                  R                  5       VVs0 s H  u  pX!_M	     nnnU R                  R                  n[        TU ]  X45      nU$ s  snnf rÎ   )r¥   rÆ   rl   ro   r  r  r  s         €r�   r  ÚGGUFGPTConverter.converted�  sZ   ø€ Ü(1°$×2IÑ2I×2PÑ2PÔ(QÔRÒ(Q™W˜Q�’Ñ(QˆÑRØ×(Ñ(×/Ñ/ˆÜ‘GÑ% eÓ4ˆ	ØÐùó Ss   ¤Ar  r  r  s   @r�   r/  r/  ˆ  s   ø† ò$ð˜9÷ õ r‘   r/  c                   ó:   • \ rS rSrS rS rS rS rS\4S jr	Sr
g	)
ÚGGUFT5Converteri”  c                 óÞ   • S/US'   [        U5      U l        [        U R                  R                  5       VVs0 s H  u  p#X2_M	     snnU l        U R                  U l        0 U l        g s  snnf ©Nz
dummy textro   )r“   rÅ   r¥   rl   Útoken2idrÆ   rÇ   )r®   rË   r±   r°   s       r�   r»   ÚGGUFT5Converter.__init__•  s_   € à$0 >ˆ�xÑ ä*¨>Ó:ˆŒ
Ü*3°D·J±J×4EÑ4EÔ*FÔGÒ*F¡$ !˜šÑ*FÒGˆŒØ"&§*¡*ˆÔØ!#ˆÕùó Hs   ¹A)c                 óT   • [        [        UR                  UR                  5      5      $ rÎ   rÏ   rÑ   s     r�   r˜   ÚGGUFT5Converter.vocabž  rÓ   r‘   c                 ó  • [        U R                  SS5      (       ae  / n[        U R                  SS5      (       a  U[        R                  " SS9/-  nU[        R                  " SSS9/-  n[        R
                  " U5      $ g )Nrâ   Trp   rñ   )ÚprependrŸ   )Úpatternrö   )rÈ   rÆ   r   ÚPrependrú   rý   )r®   rÅ   r   s      r�   r  ÚGGUFT5Converter.normalizer¡  sx   € Ü�4×*Ñ*¨H°d×;Ñ;ØˆHÜ�t×.Ñ.Ð0BÀD×IÑIØœ[×0Ò0¸Ñ?Ð@Ñ@�Øœ×,Ò,°SÀ%ÑHÐIÑIˆHÜ×'Ò'¨Ó1Ð1Ør‘   c                 óV   • [         R                  " SS// SQSU R                  S   4/S9$ )Nú$Ar$  )rB  r$  z$Br$  )ÚsingleÚpairrë   )r   ÚTemplateProcessingr8  )r®   s    r�   r  ÚGGUFT5Converter.post_processorª  s5   € Ü×,Ò,Ø˜&�>Ú-à˜Ÿ™ vÑ.Ð/ðñ
ð 	
r‘   r  c                 óØ  • U R                  U R                  5      n[        [        UU R                  R                  SS95      nU R                  U R                  5      nUb  X2l        SnSn[        U R                  S5      (       a  U R                  R                  nU R                  XE5      nUb  Xbl	        U R                  XE5      Ul
        U R                  5       nU(       a  Xrl        U$ )NF©Úunk_idrÛ   rñ   Trp   )r˜   rÅ   r   r
   r@   r  r¤   rÆ   rp   r  r  r  )r®   rç   r=   r  rÿ   rp   r  r  s           r�   r  ÚGGUFT5Converter.converted³  sÔ   € Ø—z‘z $§*¡*Ó-ˆÜÜØØ—z‘z×.Ñ.Ø#ñó
ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÑ!Ø#-Ô àˆØÐÜ�4×*Ñ*Ð,>×?Ñ?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;ÓIˆØÑ$Ø&3Ô#à ŸL™L¨ÓGˆ	ÔØ×,Ñ,Ó.ˆÞØ'5Ô$àÐr‘   )rÇ   rÆ   rÅ   r8  N)r½   r¾   r¿   rÀ   r»   r˜   r  r  r   r  rÁ   r–   r‘   r�   r5  r5  ”  s"   † ò$ò5òò
ð˜9÷ r‘   r5  c                   ó:   • \ rS rSrS rS rS rS rS\4S jr	Sr
g	)
ÚGGUFGemmaConverteriÓ  c                 ó`   • S/US'   [        U5      U l        U R                  U l        0 U l        g r7  r  rÊ   s     r�   r»   ÚGGUFGemmaConverter.__init__Ô  s.   € à$0 >ˆ�xÑ ä*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÕr‘   c                 óV  • [        [        UR                  UR                  5      5      n/ nU Hw  u  pEUS:X  a  UR	                  SU45        M   SU;   a?  [        UR                  5       5      S:X  a"  S[        U5      -  nUR	                  Xe45        Me  UR	                  XE45        My     U$ )Nz<0x09>Ú	rŸ   r   rñ   )r„   rÐ   rl   rm   r©   r…   Ústrip)r®   rÅ   Úoriginal_vocabÚupdated_vocabÚtokenÚscoreÚunderscoress          r�   r˜   ÚGGUFGemmaConverter.vocabÜ  s–   € Üœc %§,¡,°·±Ó=Ó>ˆØˆã*‰LˆEØ˜Ó Ø×$Ñ$ d¨E ]Ö3Ø˜“¤# e§k¡k£mÓ"4¸Ó"9Ø#¤c¨%£jÑ0�Ø×$Ñ$ kÐ%9Ö:à×$Ñ$ e ^Ö4ñ +ð Ðr‘   c                 ó0   • [         R                  " SS5      $ )NrŸ   rñ   )r   rú   rÑ   s     r�   r  ÚGGUFGemmaConverter.normalizerë  s   € Ü×"Ò" 3¨Ó.Ð.r‘   c                 óî   • [         R                  " SS5      [         R                  " 5       [         R                  " 5       /nU(       a  U[         R                  " SSS9/-  n[         R
                  " U5      $ )Nrñ   rŸ   rw   rõ   )r   rú   rø   rù   rü   rý   rþ   s       r�   r  ÚGGUFGemmaConverter.decoderî  s\   € ä×Ò˜U CÓ(Ü×!Ò!Ó#Ü�MŠM‹Oð
ˆö ØœŸš°¸!Ñ<Ð=Ñ=ˆHÜ× Ò  Ó*Ð*r‘   r  c                 ó²  • U R                  U R                  5      n[        [        UU R                  R                  U R
                  S95      nU R                  U R                  5      nUb  X2l        SnSn[        U R                  S5      (       a  U R                  R                  nU R                  XE5      Ul
        U R                  XE5      nUb  Xbl        U$ )NrH  rñ   Trp   )r˜   rÅ   r   r
   r@   Úhandle_byte_fallbackr  r¤   rÆ   rp   r  r  )r®   rç   r=   r  rÿ   rp   r  s          r�   r  ÚGGUFGemmaConverter.convertedù  sÃ   € Ø—z‘z $§*¡*Ó-ˆÜÜØØ—z‘z×.Ñ.Ø"×7Ñ7ñó
ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÑ!Ø#-Ô àˆØÐÜ�4×*Ñ*Ð,>×?Ñ?Ø#×6Ñ6×GÑGÐà ŸL™L¨ÓGˆ	ÔØ×*Ñ*¨;ÓIˆØÑ$Ø&3Ô#àÐr‘   r-  N)r½   r¾   r¿   rÀ   r»   r˜   r  r  r   r  rÁ   r–   r‘   r�   rL  rL  Ó  s"   † ò$òò/ò	+ð˜9÷ r‘   rL  rd   re   rf   Úgemma3_textrh   )ri   Údecilmrj   r   r  c                 ó^   • U n[         U   " U5      nUR                  5       nXCR                  4$ )a  
Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

Args:
    architecture (`str`): The model architecture derived from gguf file.
    transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
        Instance of a slow tokenizer to convert in the backend tokenizer for
        [`~tokenization_utils_base.PreTrainedTokenizerFast`].

Return:
    A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
    [`~tokenization_utils_base.PreTrainedTokenizerFast`]
)ÚGGUF_TO_FAST_CONVERTERSr  rÇ   )r   rË   Útokenizer_class_nameÚ	converterÚfast_tokenizers        r�   Úconvert_gguf_tokenizerrf  ,  s7   € ð (ÐÜ'Ð(<Ò=¸nÓM€IØ×(Ñ(Ó*€NØ×6Ñ6Ð6Ð6r‘   ),Ú__doc__r   Únumpyrã   Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr	   r
   Ú r   Úconvert_slow_tokenizerr   r   r   r   r   Úutilsr   Úutils.loggingr   Ú
get_loggerr½   r¦   ÚGGUF_CONFIG_MAPPINGÚGGUF_TOKENIZER_MAPPINGÚGGUF_CONFIG_DEFAULTS_MAPPINGrŒ   r“   rÃ   r
  r  r/  r5  rL  rb  Ústrr¬   Údictrf  r–   r‘   r�   Ú<module>ru     sÉ  ðñõ
 ã ß SÕ Sß *å ß oÕ oÝ Ý  ð 
×	Ò	˜HÓ	%€ð`ØØ$Ø%ñð`ð
 Ø3Ø*Ø2Ø)à *Ø&Ø 5Ø#8Ø,:Ø"ñð`ð$ Ø3Ø*Ø2Ø)à *Ø&Ø 5Ø#8Ø,:Ø"ñð%`ð> Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñð?`ðV Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"Ø%Ø2ñðW`ðr Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"Ø+Ø2Ø*ñðs`ðP Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"Ø+ñðQ`ðj Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñðk`ðB Ø3Ø*Ø2Ø)Ø $Ø&Ø *Ø 5Ø#8Ø,:Ø"Ø%Ø2ñðC`ð` Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñða`ðx Ø+Ø+Ø!/Ø!/ñ	ðy`ðD Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñðE`ð\ Ø Ø)Ø (Ø"Ø(<ñð]`ðj 	Ø'Ø#Ø%Ø%Ø &Ø +Ø#8Ø(<Ø,LØ":Ø"ñðk`ðD Ø3Ø*Ø2Ø)Ø $Ø 5Ø#8Ø(8Ø"ñ
ðE`ðZ Ø Ø!Ø$Ø4Ø (Ø(<ñð[`ðj Ø*Ø3Ø)Ø2Ø 5Ø#8Ø(6ñðk`ð~ #Ø3Ø)Ø,@Ø*Ø(Ø&Ø.Ø-ñ
ð 4Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,6Ø"ñð 4Ø*Ø2Ø)Ø $Ø&ð !+Ø 5Ø#8Ø,:Ø$4Ø"ñð" 4Ø*Ø2Ø)Ø $Ø&ð !+Ø 5Ø#8Ø,:Ø$4Ø"ñð" (Ø#Ø%Ø%Ø &Ø +Ø#8Ø(<Ø,LØ":Ø"ñð 4Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñðØÐ3ðàÐ*ðð 	Ð2ðð 	˜Mð	ð
 	 ðð 	˜,ðð 	Ð 5ðð 	"Ð#8ðð 	 
ðð 	! $ðð 	+¨Nðð 	Ð+ðð 	Ð2ðð 	% dðð 	�lðð  	˜nð!ò]`Ð ðH	 'ØØØ'ØØ+Ø+Ø!/Ø!/Ø!3ñð )Ø"Ø+Ø+Ø!/Ø!/ññÐ ð: 	˜$ð	ð 	˜Dð	ñ Ð ò ÷0(6ñ (6ôVv˜ô vôr˜ô ô(H˜ô HôV	�}ô 	ô<�kô <ô~>˜ô >ðBØÐðàÐðð Ð#ðð Ðð	ð
 Ð#ðð Ððð Ððð Ððð Ð ðð Ððð Ð"ðð 	ˆ/ðð Ððð Ð ðð Ð ðð  Ð%ð!ð" ˆOð#ð$ Ø Ø$ò)Ð ð07¨ð 7ÀÀyÐRVÀÑAWõ 7r‘   