pyiӶ| %SrSSKrSSKrSSKrSSKrSSKJr SSKJr SSK J r SSK J r SSK JrJr SS KJr SS KJr SS KJrJrJrJrJr SS KJr S SKJr SSKJr SSK J!r!J"r"J#r#J$r$J%r% \"5(aSSK&J'r' OSr'\"5(aSSK(J)r) OSr)\RT"\+5r,0r-\.\/\0\4\1S'0r2\.\/\0\4\1S'\\/\/S-4"/S\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS \"5(aSOS4PS!\"5(aS"OS4PS#\"5(aS$OS4PS%PS&\"5(aSOS4PS'\"5(aS(OS4PS)PS*PS+\"5(aS,OS4PS-\"5(aS.OS4PS/PS0\"5(aS1OS4PS2PS3\"5(aSOS4PS4\"5(aS5OS4PS6PS7\"5(aSOS4PS8PS9\"5(aS:OS4PS;PS<\"5(aSOS4PS=PS>\"5(aSOS4PS?\"5(aSOS4PS@PSA\"5(aSBOS4PSC\"5(aS5OS4PSD\"5(aSOS4PSE\"5(aSOS4PSF\"5(aSOS4PSG\"5(aSOS4PSH\"5(aSIOS4PSJPSKPSLPSMPSN\"5(aS5OS4PSO\"5(aSPOS4PSQ\"5(aSROS4PSSPST\"5(aSOS4PSU\"5(aSVOS4PSW\"5(aSOS4PSX\"5(aS5OS4PSY\"5(aSOS4PSZPS[\"5(aS\OS4PS]\"5(aS^OS4PS_PS`\"5(aSOS4PSa\"5(aS5OS4PSb\"5(aScOS4PSd\"5(aSeOS4PSfPSg\"5(aShOS4PSi\"5(aSjOS4PSk\"5(aSjOS4PSl\"5(aSjOS4PSm\"5(aSjOS4PSn\"5(aSjOS4PSo\"5(aSjOS4PSp\"5(aSOS4PSq\"5(aSrOS4PSs\"5(aSrOS4PSt\"5(aSrOS4PSu\"5(aSrOS4PSv\"5(aSrOS4PSw\"5(aSrOS4PSx\"5(aSrOS4PSy\"5(aSrOS4PSz\"5(aSrOS4PS{\"5(aS|OS4PS}\"5(aS5OS4PS~\"5(aS5OS4PS\"5(aS5OS4PS\"5(aS\OS4PSPS\"5(aS5OS4PSPSPSPSPS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PSPSPS\"5(aSOS4PS\"5(aSOS4PS\"5(aS5OS4PS\"5(aS5OS4PS\"5(aSOS4PS\"5(aS5OS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aS"OS4PS\"5(aS"OS4PSPS\"5(aSOS4PS\"5(aSOS4PS\"5(aS\OS4PS\"5(aS\OS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PSPS\"5(aSOS4PS\"5(aSOS4PSPS\"5(aSrOS4PS\ "5(aSO\"5(aSrOS4PS\ "5(aSO\"5(aSrOS4PS\ "5(aSO\"5(aSrOS4PS\ "5(aSO\"5(aSrOS4PS\ "5(aSO\"5(aSrOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aS\OS4PSPS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PSPS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aS\OS4PS\"5(aS\OS4PS\"5(aSrOS4PS\"5(aSrOS4PS\"5(aS\OS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aS5OS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aS.OS4PS\"5(aS.OS4PSPS\"5(aS5OS4PSPS\"5(aSOS4PS\ "5(aSO\"5(aSrOS4PS\"5(aSOS4PS\"5(aSOS4PSPS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PS\"5(aSOS4PGS\"5(aSOS4PGS\"5(aSOS4PGSPGS\"5(aSOS4PGS\"5(aSjOS4PGS\"5(aGSOS4PGS\"5(aGSOS4PGS \"5(aSOS4PGS PGS PGS PGS \"5(aGSOS4PGS\"5(aS\OS4PGS\"5(aSOS4PGS\"5(aSOS4PGS\"5(aGSOS4PGS\"5(aGSOS4PGS\"5(aSjOS4PGS\"5(aGSOS4PGS\"5(aGSOS4PGS\"5(aGSOS4PGS\"5(aGSOS4PGSPGS\"5(aSOS4PGS \"5(aS\OS4PGS!\"5(aS5OS4PGS"\"5(aSOS4PGS#\"5(aSOS4PGS$\"5(aSjOS4PGS%PGS&\"5(aSOS4PGS'\"5(aSOS4PGS(\"5(aGS)OS4PGS*\"5(aSOS4PGS+PGS,PGS-\"5(aSOS4PGS.\"5(aSOS4PGS/PGS0\ "5(aSO\"5(aSrOS4PGS1\ "5(aSO\"5(aSrOS4PGS2PGS3PGS4PGS5PGS6\"5(aGS7OS4PGS8\"5(aSOS4PGS9\"5(aGS:OS4PGS;PGS<\"5(aSOS4PGS=\"5(aSOS4PGS>\"5(aGS?OS4PGS@\"5(aS\OS4PGSA\"5(aSOS4PGSB\"5(aSOS4P5r31GSCkr4\5\/\1GSD'\4Hr6\6\3;dM \"5(aSrOS\3\6'M \"\!\35r7\!Rp"5VVs0sHupX_M snnr9GSEr:GSFr;GSG\/GSH\0\S-4GSIjr<GSVGSJ\/\Rz\/-GSK\/\Rz\/-S-GSL\>GSM\.\/\/4S-GSN\>\/-S-GSO\/S-GSP\>GSQ\/GSH\.\/\44GSRjjr?"GSSGST5r@GSUGST/rAgs snnf(WzAuto Tokenizer class.N) OrderedDict)Any)is_mistral_common_available)PreTrainedConfig)get_class_from_dynamic_moduleresolve_trust_remote_code)load_gguf_checkpoint)TOKENIZER_CONFIG_FILE)extract_commit_hashis_g2p_en_availableis_sentencepiece_availableis_tokenizers_availablelogging) cached_file)EncoderDecoderConfig)_LazyAutoMapping)CONFIG_MAPPING_NAMES AutoConfigconfig_class_to_model_typemodel_type_to_module_name!replace_list_option_in_docstrings)TokenizersBackend)SentencePieceBackendREGISTERED_TOKENIZER_CLASSESREGISTERED_FAST_ALIASESaimv2 CLIPTokenizeralbertAlbertTokenizeralign BertTokenizeraudioflamingo3Qwen2Tokenizer aya_visionCohereTokenizerbarkbartRobertaTokenizerbarthezBarthezTokenizer)bartphoBartphoTokenizerbertzbert-generationBertGenerationTokenizer)z bert-japaneseBertJapaneseTokenizer)bertweetBertweetTokenizerbig_birdBigBirdTokenizerbigbird_pegasusPegasusTokenizer)biogptBioGptTokenizer blenderbotBlenderbotTokenizer)zblenderbot-smallBlenderbotSmallTokenizerblipzblip-2 GPT2Tokenizer) bridgetowerr+bros)byt5 ByT5Tokenizer camembertCamembertTokenizer)canineCanineTokenizer chinese_clip)clapr+clipclipseg)clvp ClvpTokenizer code_llamaCodeLlamaTokenizercodegencoherecohere2colqwen2convbertcpm CpmTokenizer)cpmantCpmAntTokenizer)ctrl CTRLTokenizer)zdata2vec-audioWav2Vec2CTCTokenizer)z data2vec-textr+dbrxdebertaDebertaTokenizerz deberta-v2DebertaV2Tokenizer)dia DiaTokenizer distilbertdprDPRQuestionEncoderTokenizerelectraemu3ernie)esm EsmTokenizer falcon_mambaGPTNeoXTokenizerfastspeech2_conformerFastSpeech2ConformerTokenizer)flaubertFlaubertTokenizerflava flex_olmo florence2 BartTokenizerfnet FNetTokenizer)fsmt FSMTTokenizerfunnelFunnelTokenizergemmaGemmaTokenizergemma2gemma3 gemma3_textgemma3n gemma3n_textgitglmrglm4glm4_moe glm4_moe_liteglm4v glm4v_moe glm_imageglmasrgot_ocr2zgpt-sw3GPTSw3Tokenizergpt2 gpt_bigcodegpt_neogpt_neox)gpt_neox_japaneseGPTNeoXJapaneseTokenizergptj)graniter?) granitemoer?)granitemoehybridr?)granitemoesharedr?zgrounding-dinogroupvitherbertHerbertTokenizer)hubertr[)ibertr+ideficsLlamaTokenizeridefics2 instructblipinstructblipvideointernvljais2jina_embeddings_v3XLMRobertaTokenizerzkosmos-2lasr_ctc LasrTokenizer lasr_encoderlayoutlm layoutlmv2LayoutLMv2Tokenizer layoutlmv3LayoutLMv3Tokenizer layoutxlmLayoutXLMTokenizerled LEDTokenizer lighton_ocrQwen2TokenizerFastlilt longformer)luke LukeTokenizerlxmertLxmertTokenizerm2m_100M2M100Tokenizermambamamba2marianMarianTokenizermarkuplmMarkupLMTokenizermbartMBartTokenizermbart50MBart50Tokenizer)megar+z megatron-bert metaclip_2)zmgp-strMgpstrTokenizer minicpmv4_6 ministralMistralCommonBackend ministral3mistralmistral3mixtralmlukeMLukeTokenizerzmm-grounding-dino mobilebertMobileBertTokenizermpnetMPNetTokenizermpt)mrar+mt5 T5Tokenizermusicgenmusicgen_melodymvp MvpTokenizer)myt5 MyT5Tokenizernezhanllb NllbTokenizerznllb-moe nomic_bertnougatNougatTokenizer nystromformerolmoolmo2olmo3 olmo_hybridolmoez omdet-turbo oneformerz openai-gptOpenAIGPTTokenizeroptovis2owlv2owlvitpegasus pegasus_x) perceiverPerceiverTokenizerphi)phobertPhobertTokenizer pix2structpixtralplbartPLBartTokenizer pp_formulanet) prophetnetProphetNetTokenizerqdqbert qianfan_ocrqwen2 qwen2_5_omni qwen2_5_vl qwen2_audio qwen2_moeqwen2_vlqwen3qwen3_5Qwen3_5Tokenizer qwen3_5_moe qwen3_moe qwen3_nextqwen3_omni_moeqwen3_vl qwen3_vl_moe)rag RagTokenizerrealmrecurrent_gemmareformerReformerTokenizerrembertRemBertTokenizer retribert)robertar+)zroberta-prelayernormr+)roc_bertRoCBertTokenizerroformerRoFormerTokenizerrwkvsam3 sam3_video seamless_m4tSeamlessM4TTokenizerseamless_m4t_v2 shieldgemma2siglipSiglipTokenizersiglip2Siglip2Tokenizerspeech_to_textSpeech2TextTokenizerspeecht5SpeechT5Tokenizer)splinterSplinterTokenizer squeezebertstablelm starcoder2switch_transformerst5t5gemma)tapasTapasTokenizertrocrtvpudop UdopTokenizerumt5) unispeechr[)z unispeech-satr[vilt visual_bert)vits VitsTokenizervoxtralvoxtral_realtime)wav2vec2r[)z wav2vec2-bertr[)zwav2vec2-conformerr[)wav2vec2_phonemeWav2Vec2PhonemeCTCTokenizerwhisperWhisperTokenizerxclipxglm XGLMTokenizer)xlm XLMTokenizerz xlm-robertazxlm-roberta-xlxlnetXLNetTokenizerxlstmxmodyoso>%fuyuphi3jambajanusllavamolmonvfp4rarcticchatlmmolmo2phi3_vphimoeopencuaopenvlastep3p5minicpmvnemotronstep3_vlvipllava chameleon internlm2 cohere_asr h2ovl_chat llava_next minimax_m2 modernbert deepseek_v2 deepseek_v3 deepseek_v4 deepseek_vl deepseek_ocr deepseek_ocr2 internvl_chatdeepseek_vl_v2hyperclovax_vlmdeepseek_vl_hybrid)MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASScz[USSS9n[R"U5sSSS5 $!,(df  g=f)z*Loads a vocabulary file into a dictionary.rutf-8encodingN)openjsonload) vocab_filereaders g/home/mande/repo/quber/.venv/lib/python3.13/site-packages/transformers/models/auto/tokenization_auto.py load_vocabrs( j# 0Fyy  1 0 0s, :c/n[USSS9nUH\nUR5nU(dMURS5(aM4UR[ UR 555 M^ SSS5 U$!,(df  U$=f)z Loads a merges file into a list.rxryrz#N)r|strip startswithappendtuplesplit) merges_filemergesrlines r load_mergesrsp F k3 1VD::BloomTokenizerBloomTokenizerFastr)rrrrrrrArz.tokenization_mistral_common transformers.ztransformers.models __module__Fast_fast__name__)rrrTOKENIZER_MAPPING_NAMESitemsr importlib import_modulegetattrsysmodulessetattrr setdefaultAttributeErrorTOKENIZER_MAPPING_extra_contentvalueshasattrendswithtokenizer_class_from_name) r module_nametokenizer_classmoduleresultsubmodbase_mod tokenizer main_modules rrrs==  ,,&z2211+J77((  )@(E(E(G$  (3K@Krr"88"001OQ_`"001[M1BDYZ  4%flDAAFAvQTQ\Q\G\"{{62HHoo&>GKK**6G+;XF #)H*'55<<> 9j$ /: = ?)).9K{''{//6""(CR99 %"  s)BG GGpretrained_model_name_or_path cache_dirforce_downloadproxiestokenrevisionlocal_files_only subfolderc "URS5n [U[UUUUUUUSSSU S9 n U c[R S5 0$[ X5n [ U SS9n [R"U 5n SSS5 U W S'U $!,(df  N=f)a Loads the tokenizer configuration from a pretrained model tokenizer configuration. Args: pretrained_model_name_or_path (`str` or `os.PathLike`): This can be either: - a string, the *model id* of a pretrained model configuration hosted inside a model repo on huggingface.co. - a path to a *directory* containing a configuration file saved using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. cache_dir (`str` or `os.PathLike`, *optional*): Path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used. force_download (`bool`, *optional*, defaults to `False`): Whether or not to force to (re-)download the configuration files and override the cached versions if they exist. proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated when running `hf auth login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git. local_files_only (`bool`, *optional*, defaults to `False`): If `True`, will only try to load the tokenizer configuration from local files. subfolder (`str`, *optional*, defaults to `""`): In case the tokenizer config is located inside a subfolder of the model repo on huggingface.co, you can specify the folder name here. Passing `token=True` is required when you want to use a private model. Returns: `dict`: The configuration of the tokenizer. Examples: ```python # Download configuration from huggingface.co and cache. tokenizer_config = get_tokenizer_config("google-bert/bert-base-uncased") # This model does not have a tokenizer config so the result will be an empty dict. tokenizer_config = get_tokenizer_config("FacebookAI/xlm-roberta-base") # Save a pretrained tokenizer locally and you can reload its config from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased") tokenizer.save_pretrained("tokenizer-test") tokenizer_config = get_tokenizer_config("tokenizer-test") ``` _commit_hashF) rrrrrrr _raise_exceptions_for_gated_repo%_raise_exceptions_for_missing_entries'_raise_exceptions_for_connection_errorsrNz\Could not locate the tokenizer configuration file, will try to use the model config instead.ryrz) getrr loggerinfor r|r}r~) rrrrrrrrkwargs commit_hashresolved_config_filerrs rget_tokenizer_configrsJ**^,K&%%))..305 # rs %&:HK "W 56" 6(F> M 6 5s B Bch\rSrSrSrSr\\"\5S\ \ -4Sj55r \ S Sj5r Srg) AutoTokenizeri4a This is a generic tokenizer class that will be instantiated as one of the tokenizer classes of the library when created with the [`AutoTokenizer.from_pretrained`] class method. This class cannot be instantiated directly using `__init__()` (throws an error). c[S5e)Nz}AutoTokenizer is designed to be instantiated using the `AutoTokenizer.from_pretrained(pretrained_model_name_or_path)` method.)OSError)selfs r__init__AutoTokenizer.__init__<s _  rc BURSS5nSUS'URSS5nURSS5nURSS5nURS5nUbx[RUS5n U c,[S US S R S [55S 35e[ U 5n U c[SU S35eU R "U/UQ70UD6$U(a0[X40UD6n [U SS9Sn [R"S+0U D6nOUc[R "U4SU0UD6nURn [U40UD6nURSS5nSnSU;a9[US[ ["45(aUSnOUSRSS5nUcUbU bU S:wa[RU 5b[RU 5R%S5UR%S5:wa[RU 5R%S5nUS;aBU [&;aUOUn[ U5n U b&U R(S;aU R "U/UQ70UD6$[*b[*R "U/UQ70UD6$[SUS35eSU;aUSUS'U(aUR-S5(aUSSnUSLn[/U5[0;=(d/ USL=(a$ [ U5SL=(d [ US-5SLnU=(ab [/U5[0;=(aI USL=(a> [ U5=(d [ US-5R2R5S5(+nU(aU [&;aSnSnU(a<USbUSnOUSnSU;aUR7S5SnOSn[9XqUUU5nU(avU(aoU(dhU(a[ UR%S55 [;WU40UD6n URS S5nU R=5 U R "U/UQ7SU0UD6$UblUn[ U5n U c$UR-S5(d[ US-5n U bU R(S!:Xa[*n U c[*n U R "U/UQ70UD6$[?USS5(aNUR@nS"U;aUR-S5(aUSSn[ U5n U R "U/UQ70UD6$[U[B5(a{[/URD5[/URF5LaD[HRKS#URFRLS$URDRLS%35 URFn[O[/U5R(5=(d [?US&S5nUb<[0R[/U5[*5n U bU R "U/UQ70UD6$URSS5nUbUS':waUR-S5(aUSSn[ U5n U c$UR-S5(d[ US-5n U bU R(S!:Xa[*n U c[*n U R "U/UQ70UD6$[S(URLS)S R S*[055S 35e![[4a [R "U40UD6nGNf=f),a Instantiate one of the tokenizer classes of the library from a pretrained model vocabulary. The tokenizer class to instantiate is selected based on the `model_type` property of the config object (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by falling back to using pattern matching on `pretrained_model_name_or_path`: List options Params: pretrained_model_name_or_path (`str` or `os.PathLike`): Can be either: - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co. - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. - a path to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (like Bert or XLNet), e.g.: `./my_model_directory/vocab.txt`. (Not applicable to all derived classes) inputs (additional positional arguments, *optional*): Will be passed along to the Tokenizer `__init__()` method. config ([`PreTrainedConfig`], *optional*) The configuration object used to determine the tokenizer class to instantiate. cache_dir (`str` or `os.PathLike`, *optional*): Path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used. force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download the model weights and configuration files and override the cached versions if they exist. proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. revision (`str`, *optional*, defaults to `"main"`): The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git. subfolder (`str`, *optional*): In case the relevant files are located inside a subfolder of the model repo on huggingface.co (e.g. for facebook/rag-token-base), specify it here. tokenizer_type (`str`, *optional*): Tokenizer type to be loaded. backend (`str`, *optional*, defaults to `"tokenizers"`): Backend to use for tokenization. Valid options are: - `"tokenizers"`: Use the HuggingFace tokenizers library backend (default) - `"sentencepiece"`: Use the SentencePiece backend trust_remote_code (`bool`, *optional*, defaults to `False`): Whether or not to allow for custom models defined on the Hub in their own modeling files. This option should only be set to `True` for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine. kwargs (additional keyword arguments, *optional*): Will be passed to the Tokenizer `__init__()` method. Can be used to set special tokens like `bos_token`, `eos_token`, `unk_token`, `sep_token`, `pad_token`, `cls_token`, `mask_token`, `additional_special_tokens`. See parameters in the `__init__()` for more details. Examples: ```python >>> from transformers import AutoTokenizer >>> # Download vocabulary from huggingface.co and cache. >>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased") >>> # Download vocabulary from huggingface.co (user-uploaded) and cache. >>> tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-german-cased") >>> # If vocabulary files are in a directory (e.g. tokenizer was saved using *save_pretrained('./test/saved_model/')*) >>> # tokenizer = AutoTokenizer.from_pretrained("./test/bert_saved_model/") >>> # Download vocabulary from huggingface.co and define model-specific arguments >>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/roberta-base", add_prefix_space=True) >>> # Explicitly use the tokenizers backend >>> tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer", backend="tokenizers") >>> # Explicitly use the sentencepiece backend >>> tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer", backend="sentencepiece") ```configNT _from_autouse_fasttokenizer_typetrust_remote_code gguf_filezPassed `tokenizer_type` z3 does not exist. `tokenizer_type` should be one of z, c3$# UHov M g7fN.0cs r 0AutoTokenizer.from_pretrained..s D,Cq,CsrzTokenizer class z is not currently imported.F)return_tensorsrauto_maprr)r PythonBackendPreTrainedTokenizerFastzTokenizer class 'zf' specified in the tokenizer config was not found. The tokenizer may need to be converted or re-saved.rrz transformers.rrz-- code_revisionrrz The encoder model config class: z3 is different from the decoder model config class: z. It is not recommended to use the `AutoTokenizer.from_pretrained()` method in this case. Please use the encoder and decoder specific tokenizer classes. model_typerz!Unrecognized configuration class z8 to build an AutoTokenizer. Model type should be one of c38# UHoRv M g7fr)rrs rrrWs4[IZAZZIZsr)(poprr ValueErrorjoinrfrom_pretrainedrr r for_modelrrrr isinstancerlist removesuffixrvrrrtyperrrrr rregister_for_auto_classrrrdecoderencoderrwarning __class__r)clsrinputsrr_rrrtokenizer_class_namer gguf_path config_dictconfig_model_typetokenizer_configtokenizer_config_classtokenizer_auto_mapregistered_class_namerhas_remote_codehas_local_codeexplicit_local_code class_ref upstream_repotokenizer_class_candidate_classrs rrAutoTokenizer.from_pretrainedBsdHd+#| JJz4 ($4d;"JJ':DAJJ{+   %#:#>#>~t#T #+ .~.>>qyy D,C DDEQH 88LMO& #34H3IId!eff"223PdSYd]cd d #$AWPVWI.yOPXYK))8K8F ^ c#331EVZ` #--00MXQWX!1!5!56G!N" ) )*:6 FF%5j%A"%5j%A%E%EoW[%\"  &&2!-!R''++,=>J(,,->?LLVT&33F;=%<$?$?@Q$R$_$_`f$g !$,mm),UU*/ # " !&<&E&Ef&M&M%;CR%@ ",D8f):: "$ . )*@AMZ,-Cf-LMUYY   V $55 'd29-.DER01G&1PQ*ZZ89  04]]#O!%  !!$0.q1 .q1 y ) 5a 8 $ 9!.Racp!  09L%)*@*M*Mf*UV;IGdohnoO ?D1A  3 3 5"22-06J[_e $ /(> %78QRO&/H/Q/QRX/Y/Y";?22 ^^F/V 0E0EFm'RXZfhlJm  !/33DLBSTO*&667ThW]haghh"2!5!56G!N ! -%)<?T 1 1T @T lkpUUrrr)NFNNNFr)Brrr}osr collectionsrtypingrtransformers.utils.import_utilsrconfiguration_utilsrdynamic_module_utilsrr modeling_gguf_pytorch_utilsr tokenization_utils_baser utilsr r rrr utils.hubrencoder_decoderr auto_factoryrconfiguration_autorrrrrtokenization_utils_tokenizersr tokenization_utils_sentencepiecer get_loggerrrrdictstrr__annotations__rrrvsetrrrCONFIG_TO_TYPErrrPathLikeboolrr__all__)kvs00rr.s #G3\?<%2*BH   H %68d3S >2702c49n-2%c3:o6V %<%>%>/DIV (?(A(A$tLV %<%>%>/DIV /F/H/H+dS V ,C,E,E(4P V $;$=$=4H V '>'@'@#dKV *A*C*C&NV (V $;$=$=4HV 9S9U9U5[_`V 3V *V +B+D+D'$OV 2I2K2K.QUVV &!V" 0G0I0I,tT#V$ 9%V& $;$=$=4H'V( &=&?&??TJ)V* ,+V, $;$=$=4H-V. "/V0 .E.G.G*TR1V2 &3V4 ,C,E,E4P5V6 %7V8 $;$=$=4H9V: '>'@'@OdK;V< "=V> /F/H/H+dS?V@ '>'@'@OdKAVB (?(A(A$tLCVD )@)B)B%MEVF )@)B)B%MGVH (?(A(A_tLIVJ "9";";FKVL &MVN "OVP 3QVR .SVT $;$=$=4HUVV *A*C*C&NWVX /F/H/H+dSYVZ [V\ *A*C*CN]V^ 1H1J1J-PTU_V` '>'@'@OdKaVb $;$=$=4HcVd %<%>%>/DIeVf gVh /F/H/H+dSiVj !EXEZEZ"A`dekVl *mVn %<%>%>/DIoVp )@)B)BoMqVr )@)B)BoMsVt $;$=$=4HuVv "wVx (?(A(A$tLyVz &=&?&?"TJ{V| '>'@'@#dK}V~ '>'@'@#dKV@ ,C,E,E(4PAVB (?(A(A$tLCVD -D-F-F)DQEVF #:#<#<$GGVH '>'@'@#dKIVJ (?(A(A$tLKVL ,C,E,E(4PMVN 1H1J1J-PTUOVP )@)B)B%MQVR -D-F-F)DQSVT -D-F-F)DQUVV *A*C*C&NWVX ,C,E,E(4PYVZ )C)E)E%4P[V\ $;$=$=4H]V^ +B+D+D$O_V` '>'@'@OdKaVb +B+D+D'$OcVd :eVf $;$=$=4HgVh %iVj (kVl .mVn .oVp .E.G.G?TRqVr (?(A(A_tLsVt *A*C*C&NuVv +wVx &yVz (?(A(A$tL{V| )@)B)B%M}V~ ,C,E,E4PV@ 1H1J1JoPTUAVB )@)B)B%MCVD %<%>%>/DIEVF 8O8Q8Q4W[\GVH .E.G.G*TRIVJ (?(A(A_tLKVL ,C,E,E4PMVN (?(A(A_tLOVP 0G0I0I,tTQVR 0G0I0I,tTSVT .E.G.G*TRUVV "9";";FWVX 0G0I0I,tTYVZ '>'@'@#dK[V\ -D-F-F)DQ]V^ "_V` (?(A(A$tLaVb )C)E)E%4PcVd (?(A(A$tLeVf )@)B)B%MgVh (B(D(D$$OiVj ,C,E,E(4PkVl &=&?&?"TJmVn *A*C*C&NoVp %qVr -D-F-F/DQsVt 0G0I0I,tTuVv 'wVx /F/H/H+dSyV| *,, #)@)B)B%  {VH *,, #)@)B)B%  GVT *,, #)@)B)B%  SV` *,, #)@)B)B%  _Vl *,, #)@)B)B%  kVv &@&B&B"MwVx 1H1J1JoPTUyVz 0G0I0I,tT{V| &=&?&?"TJ}V~ &=&?&?"TJV@ $AVB !8!:!: ECVD &=&?&?]TJEVF -D-F-FMDQGVH "9";";FIVJ "KVL %<%>%>/DIMVN $;$=$=4HOVP (?(A(A_tLQVR *A*C*CNSVT (?(A(A$tLUVV /F/H/H+dSWVX '>'@'@#dKYVZ (?(A(A$tL[V\ )@)B)B%M]V^ /F/H/H+dS_V` (?(A(A$tLaVb +B+D+D$OcVd )@)B)BoMeVf /F/H/H+dSgVh #:#<#<$GiVj &=&?&?"TJkVl %<%>%>/DImVn &=&?&??TJoVp *A*C*C&NqVr ,C,E,E(4PsVt ,uVv #:#<#<$GwVx (yVz (?(A(A}tL{V~ *,, #)@)B)B%  }VH (?(A(A$tLIVJ /F/H/H+dSKVL .MVN '>'@'@OdKOVP ,C,E,E(4PQVR &=&?&?"TJSVT -D-F-F)DQUVV +B+D+D'$OWVX ,C,E,E(4PYVZ *A*C*C&N[V\ )@)B)B%M]V^ &=&?&?"TJ_V` *A*C*C&NaVb .E.G.G*TRcVd *A*C*C&NeVf +B+D+D'$OgVh /F/H/H+dSiVj )@)B)B%MkVl -D-F-F)DQmVn oVp %<%>%>/DIqVr 0G0I0I,tTsVt ,C,E,E(4PuVv *A*C*C&NwVx )@)B)BoMyVz ({V| 5}V~ )V@ ,C,E,E(4PAVB '>'@'@#dKCVD $;$=$=4HEVF *A*C*CNGVH 3J3L3L/RVWIVJ 6M6O6O2UYZKVL -D-F-F)DQMVN (B(D(D$$OOVP *A*C*C&NQVR 5O5Q5Q1W[\SVT ,F,H,H(dSUVV *WVX +B+D+D$OYVZ +B+D+D'$O[V\ *A*C*CN]V^ 1H1J1J PTU_V`  7 9 9}tDaVb (?(A(A$tLcVd $eVf +B+D+D'$OgVh #:#<#<$GiVj $;$=$=4HkVl "9";";FmVn .oVp 2qVr $;$=$=4HsVt +B+D+D$OuVv "wVz *,, #)@)B)B%  yVF *,, #)@)B)B%  EVP -QVR 2SVT 7UVV %>/DI[V\ $;$=$=4H]V^ _V` 1H1J1J-PTUaVb 4K4M4M0SWXcVd &=&?&?"TJeVf (?(A(A$tLgVh *A*C*C&NiVj &=&?&?"TJkVXx&7)3s8&P