ó
    >:jÑ” ã                  ó,  • S SK Jr  S SKrS SKrS SKrS SKrS SKrS SKJr  S SK	J
r
Jr  S SKJr  S SKJr  S SKJrJrJrJr  S SKrS SKrS SKrS SKJrJr  S S	KJrJrJr  S S
KJ r J!r!  S SK"J#r#J$r$J%r%J&r&  S SK'J(r(  S SK)J*r+  S SK,J-r-J.r.J/r/  S SKJ0r0J1r1J2r2J3r3  S SK4J5r5J6r6J7r7  S SK8J9r9  S SK:J;r;J<r<  S SK=J>r>J?r?  S SK@JArA  S SKBJCrC  S SKDJErE  S SKFJGrGJHrHJIrI  SSKJJKrK  SSKLJMrM  SSKNJOrOJPrPJQrQ  SSKRJSrSJTrTJUrUJVrVJWrWJXrXJYrYJZrZJ[r[J\r\J]r]J^r^J_r_J`r`JaraJbrb   " S S\9\RÆ                  RÈ                  5      re " S S \e5      rf " S! S"\e5      rg " S# S$\e5      rh " S% S&\e5      ri " S' S(\e5      rj " S) S*\e5      rk\ " S+ S,5      5       rlS1S- jrm\ " S. S/5      5       rnS2S0 jrog)3é    )ÚannotationsN)ÚSequence)ÚcontextmanagerÚnullcontext)Údeepcopy)Ú	dataclass)ÚAnyÚLiteralÚOptionalÚUnion)Údispatch_modelÚinfer_auto_device_map)ÚAlignDevicesHookÚadd_hook_to_moduleÚremove_hook_from_submodules)Úget_balanced_memoryÚnamed_module_tensors)ÚHfFileSystemÚ	ModelCardÚModelCardDataÚhf_hub_download)Ú	safe_open)Ú	save_file)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)ÚCacheÚDynamicCacheÚEncoderDecoderCacheÚPreTrainedModel)ÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPushToHubMixin)Úget_alora_offsets_for_forwardÚget_alora_offsets_for_generate)Ú	BaseTunerÚBaseTunerLayer)ÚAuxiliaryTrainingWrapper)ÚDUMMY_MODEL_CONFIG)Úinit_empty_weights)ÚTrainableTokensWrapperÚcreate_attention_maskÚ set_additional_trainable_modulesé   )Ú__version__)Ú
PeftConfig)ÚPEFT_TYPE_TO_CONFIG_MAPPINGÚPEFT_TYPE_TO_PREFIX_MAPPINGÚPEFT_TYPE_TO_TUNER_MAPPING)ÚSAFETENSORS_WEIGHTS_NAMEÚ8TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPINGÚWEIGHTS_NAMEÚPeftTypeÚTaskTypeÚ_get_batch_sizeÚ_prepare_prompt_learning_configÚ_set_adapterÚ_set_trainableÚget_peft_model_state_dictÚid_tensor_storageÚinfer_deviceÚload_peft_weightsÚmap_cache_to_layer_device_mapÚset_peft_model_state_dictÚshift_tokens_rightc                  ó  ^ • \ rS rSrSr   S)           S*U 4S jjjr\S+S j5       r\S,S j5       r\S-S j5       r	\R                  S.S j5       r     S/               S0S jjr\       S1                     S2S	 jj5       rS3S
 jrS4S jrS4S jrS5U 4S jjr S6       S7S jjrS8S jrS9S jrS:U 4S jjr\S 5       rS;S jrS rS<S jr\S 5       rS=S jr  S>         S?S jjrS@S jr\SAS j5       rSBS jr SCS jr!\SDS j5       r"SES jr#SFS jr$      SG                 SHS  jjr%S<SIS! jjr&SJSKS" jjr'\S# 5       r(\S$ 5       r)S% r*SLS& jr+SMSNS' jjr,S(r-U =r.$ )OÚ	PeftModeléH   a.  
Base model encompassing various Peft methods.

Args:
    model ([`~transformers.PreTrainedModel`]): The base transformer model used for Peft.
    peft_config ([`PeftConfig`]): The configuration of the Peft model.
    adapter_name (`str`,  *optional*): The name of the adapter, defaults to `"default"`.
    autocast_adapter_dtype (`bool`, *optional*, defaults to `True`):
        Whether to autocast the adapter dtype. Defaults to `True`. Right now, this will only cast adapter weights
        using float16 and bfloat16 to float32, as this is typically required for stable training, and only affect
        select PEFT tuners. If set to `False`, the dtypes will stay the same as those of the corresponding layer.
    low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
        Create empty adapter weights on meta device. Useful to speed up the loading loading process.

        > [!TIP] > Don't use `low_cpu_mem_usage=True` when creating a new PEFT adapter for training.

**Attributes**:
    - **base_model** ([`torch.nn.Module`]) -- The base transformer model used for Peft.
    - **peft_config** ([`PeftConfig`]) -- The configuration of the Peft model.
    - **modules_to_save** (`list` of `str`) -- The list of sub-module names to save when
        saving the model.
    - **prompt_encoder** ([`PromptEncoder`]) -- The prompt encoder used for Peft if
        using [`PromptLearningConfig`].
    - **prompt_tokens** (`torch.Tensor`) -- The virtual prompt tokens used for Peft if
        using [`PromptLearningConfig`].
    - **transformer_backbone_name** (`str`) -- The name of the transformer
        backbone in the base model if using [`PromptLearningConfig`].
    - **word_embeddings** (`torch.nn.Embedding`) -- The word embeddings of the transformer backbone
        in the base model if using [`PromptLearningConfig`].
c                ó  >• [         TU ]  5         X0l        UR                  U l        SS1U l        UR
                  U l        U R                  (       a  X20U l        Xl        U R                  X2US9  OMS U l        [        UR                     nU(       a  [        O[        nU" 5          U" XU0U5      U l        S S S 5        [        U R                  S5      (       a  U R                  R                  X4S9  [        USS5      (       a  U R!                  U5      n[        U R                  S5      (       a@  [        U R                  R"                  S	5      (       a  S
U R                  R"                  l        SU l        g ! , (       d  f       NÈ= f)NÚadapter_namesÚalora_offsets©Úlow_cpu_mem_usageÚ_cast_adapter_dtype©Úadapter_nameÚautocast_adapter_dtypeÚis_gradient_checkpointingTÚconfigÚpretraining_tpr/   F)ÚsuperÚ__init__Úactive_adapterÚ	peft_typeÚspecial_peft_forward_argsÚis_prompt_learningÚ_is_prompt_learningÚ_peft_configÚ
base_modelÚadd_adapterr4   r+   r   ÚhasattrrM   ÚgetattrÚ(prepare_model_for_gradient_checkpointingrR   rS   Ú_adapters_disabled)	ÚselfÚmodelÚpeft_configrO   rP   rL   ÚclsÚctxÚ	__class__s	           €ÚL/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/peft_model.pyrU   ÚPeftModel.__init__h   sH  ø€ ô 	‰ÑÔØ*ÔØ$×.Ñ.ˆŒð +:¸?Ð)KˆÔ&à#.×#AÑ#AˆÔ Ø×#×#Ø!-Ð ;ˆDÔØ#ŒOØ×Ñ˜\ÐJ[ÐÒ\à $ˆDÔÜ,¨[×-BÑ-BÑCˆCÞ(9Õ$¼{ˆCÙ•Ù"% e¸KÐ-HÈ,Ó"W�”÷ ô �4—?‘?Ð$9×:Ñ:Ø�O‰O×/Ñ/Ø)ð 0ñ ô �5Ð5°t×<Ñ<Ø×AÑAÀ%ÓHˆEô
 �4—?‘? H×-Ñ-´'¸$¿/¹/×:PÑ:PÐRb×2cÑ2cØ45ˆD�O‰O×"Ñ"Ô1à"'ˆÕ÷# •ús   Â%E8Å8
Fc                óh   • U R                   (       a  U R                  $ U R                  R                  $ ©N©rZ   r[   r\   rd   ©rb   s    rh   rd   ÚPeftModel.peft_config“   s'   € à×#×#Ø×$Ñ$Ð$Ø�‰×*Ñ*Ð*ó    c                ó  •  U R                   R                  n[        U[        5      (       d$  U R                  n[        U[
        5      (       a  U/nU$ ! [         a(    U R                  n[        U[
        5      (       a  U/n U$ f = frk   )r\   Úactive_adaptersÚ
isinstanceÚlistrV   ÚstrÚAttributeError)rb   Úadapterss     rh   rq   ÚPeftModel.active_adapters™   s�   € ð	&Ø—‘×6Ñ6ˆHÜ˜h¬×-Ñ-ð  ×.Ñ.�Ü˜h¬×,Ñ,Ø (˜z�Hð
 ˆøô	 ó 	&Ø×*Ñ*ˆHÜ˜(¤C×(Ñ(Ø$˜:�øØˆð		&ús   ‚AA Á.BÂBc                óÆ   • U R                   U R                     R                  (       a  U R                  (       + $ U R                  (       + =(       d    U R                  (       + $ )a  Reflects whether the adapters are purposefully disabled (via disable_adapter) or if there
are no active adapters (enabled but inactive). They are two separate mechanisms but sometimes it is helpful to
know whether the model has any active/enabled adapter at all.
)rd   rV   rY   ra   rq   rm   s    rh   Úhas_active_enabled_adapterÚ$PeftModel.has_active_enabled_adapter¬   sJ   € ð ×Ñ˜D×/Ñ/Ñ0×C×CØ×.Ñ.Ô.Ð.à×*Ñ*Ô*×F°$×2FÑ2FÔ.FÐFro   c                óT   • U R                   (       a  Xl        g XR                  l        g rk   rl   )rb   Úvalues     rh   rd   rn   ·   s   € à×#×#Ø %Õà*/�O‰OÕ'ro   c                óî
  ^ • [         R                  R                  U5      (       a  [        SU S35      eUc$  [	        T R
                  R                  5       5      nOM[        U 4S jU 5       5      (       a3  [        S[	        T R
                  R                  5       5       SU S35      eU 4S jnU(       a&  [         R                  " US	S
9  T R                  U5        U GH�  n	T R
                  U	   n
[        T UR                  SS5      U	US9nU	S:w  a  [         R                  R                  X5      OUn[         R                  " US	S
9  U(       Ga·  U(       Ga¯  [        R                  " [        5      nUR                  5        H`  u  pï[!        U["        R$                  5      (       a  U['        U5         R)                  U5        MC  U[+        U5         R)                  U5        Mb     UR                  5        VVs0 s H  u  nn[-        U5      S:”  d  M  UU_M     nnnUR                  5        H(  u  nnUSS  H  nUU   R/                  5       UU'   M     M*     Ub7  [0        R2                  " U
5      n
S	U
l        U
R7                  U5        U" X¦X·5      nUR                  5        H0  u  nnUR9                  5       (       a  M  UR;                  5       UU'   M2     [=        U[         R                  R                  U[>        5      SS0S9  OzU(       as  Ub7  [0        R2                  " U
5      n
S	U
l        U
R7                  U5        U" X¦X·5      n["        R@                  " U[         R                  R                  U[B        5      5        U
RD                  cl  U
RF                  (       a&  T RH                  RJ                  R                  SS5      O/T RH                  RL                  RJ                  R                  SS5      U
l"        U
RN                  nS	U
l'        U
RP                  c5  T RS                  U
RF                  S9nURT                  nURV                  US.nOSnU(       aÿ  Ubì  S	U
l        U
=RX                  S-  sl,        U
RZ                  (       d  U
=R\                  S-  sl.        OU
=R\                  S-  sl.        U
R^                  (       a9  U
R^                  R                  5        VVs0 s H  u  nnUSU-  _M     snnU
l/        U
R`                  (       a9  U
R`                  R                  5        VVs0 s H  u  nnUSU-  _M     snnU
l0        U
R7                  UUS9  UU
l'        GM„     gs  snnf s  snnf s  snnf )uÉ  
This function saves the adapter model and the adapter configuration files to a directory, so that it can be
reloaded using the [`PeftModel.from_pretrained`] class method, and also used by the [`PeftModel.push_to_hub`]
method.

Args:
    save_directory (`str`):
        Directory where the adapter model and configuration files will be saved (will be created if it does not
        exist).
    safe_serialization (`bool`, *optional*):
        Whether to save the adapter files in safetensors format, defaults to `True`.
    selected_adapters (`List[str]`,  *optional*):
        A list of adapters to be saved. If `None`, will default to all adapters.
    save_embedding_layers (`Union[bool, str]`, *optional*, defaults to `"auto"`):
        If `True`, save the embedding layers in addition to adapter weights. If `auto`, checks the common
        embedding layers `peft.utils.other.EMBEDDING_LAYER_NAMES` in config's `target_modules` when available.
        and automatically sets the boolean flag. This only works for ðŸ¤— transformers models.
    is_main_process (`bool`, *optional*):
        Whether the process calling this is the main process or not. Will default to `True`. Will not save the
        checkpoint if not on the main process, which is important for multi device setups (e.g. DDP).
    path_initial_model_for_weight_conversion (`str, *optional*`):
        The path to the initialized adapter, which is obtained after initializing the model with
        PiSSA/CorDA/OLoRA and before performing any training. When `path_initial_model_for_weight_conversion`
        is not None, the difference in adapter before and after fine-tuning is calculated. This difference can
        be represented as the parameters of a standard LoRA adapter. Using this converted adapter does not
        require changes to the base model, thus conveniently allowing the use of multiple PiSSA/CorDA/OLoRA
        adapters with LoRA adapters, and the activation or deactivation of any adapters. Note that this
        conversion is not supported if `rslora` is used in combination with `rank_pattern` or `alpha_pattern`.
    kwargs (additional keyword arguments, *optional*):
        Additional keyword arguments passed along to the `push_to_hub` method.

zProvided path (z#) should be a directory, not a fileNc              3  óp   >#   • U  H+  nU[        TR                  R                  5       5      ;  v •  M-     g 7frk   )rs   rd   Úkeys)Ú.0Úselected_adapter_namerb   s     €rh   Ú	<genexpr>Ú,PeftModel.save_pretrained.<locals>.<genexpr>î   s2   øé € ð â->Ð)ð &¬T°$×2BÑ2B×2GÑ2GÓ2IÓ-JÖJÚ->ùs   ƒ36zYYou passed an invalid `selected_adapters` arguments, current supported adapter names are z - got Ú.c                ó
  >^ • T R                   (       a/  T R                  (       d  T R                  (       a  Sn[        U5      e[	        U 4S jS 5       5      (       d  [
        R                  " S5        [        R                  R                  U5      n T
R                  [        R                  R                  U5      UUS9  [        T
R                  U   R                  5      R                  5       R!                  S5      n[        T
R                  U   R                  5      R                  5       S:H  n[        T
R                  U   R                  5      R                  5       S:H  n[        T
R                  U   R                  5      R                  5       S	:H  n	U(       d  U(       d  U(       d  U	(       a  [        S
5      eT
R"                  R%                  X%U5      nT
R'                  U5        U$ ! T
R'                  U5        f = f)Nz¬Passing `path_initial_model_for_weight_conversion` to `save_pretrained` is not supported when using `rank_pattern` or `alpha_pattern` at the same time as `use_rslora=True`.c              3  óˆ   >#   • U  H7  n[        TR                  5      R                  5       R                  U5      v •  M9     g 7frk   )rt   Úinit_lora_weightsÚlowerÚ
startswith)r€   Úprefixrd   s     €rh   r‚   ÚJPeftModel.save_pretrained.<locals>.save_mutated_as_lora.<locals>.<genexpr>ÿ   s:   øé € ð âL�Fô �K×1Ñ1Ó2×8Ñ8Ó:×EÑEÀf×MÐMÚLùs   ƒ?A)ÚpissaÚcordaÚoloraÚlora_gaÚtruezz`path_initial_model_for_weight_conversion` only works for converting a PiSSA/CorDA/OLoRA/LoRA-GA adapter to a LoRA adapter)Ú	subfolderrO   rŒ   r�   rŽ   r�   z·The `init_lora_weights` parameter of the initial adapter should be set to `True`. Otherwise, `self.load_adapter` will subtract the decomposed values again based on the residual model.)Ú
use_rsloraÚrank_patternÚalpha_patternÚ
ValueErrorÚanyÚwarningsÚwarnÚosÚpathÚbasenameÚload_adapterÚdirnamert   rd   r‡   rˆ   r‰   r\   Úsubtract_mutated_initÚdelete_adapter)rd   Ú(path_initial_model_for_weight_conversionÚoutput_state_dictÚkwargsÚmsgÚinitial_adapter_nameÚis_pissaÚis_cordaÚis_oloraÚ
is_lora_garb   s   `         €rh   Úsave_mutated_as_loraÚ7PeftModel.save_pretrained.<locals>.save_mutated_as_lora÷   s¼  ù€ Ø×%×%¨;×+C×+CÀ{×G`×G`ðeð ô ! “oÐ%äô áLó÷ ñ ô —’ð%ôô $&§7¡7×#3Ñ#3Ð4\Ó#]Ð ð:Ø×!Ñ!Ü—G‘G—O‘OÐ$LÓMØ2Ø!5ð "ñ ô
 ˜t×/Ñ/Ð0DÑE×WÑWÓX×^Ñ^Ó`×kÑkÐlsÓt�Ü˜t×/Ñ/Ð0DÑE×WÑWÓX×^Ñ^Ó`ÐdkÑk�Ü˜t×/Ñ/Ð0DÑE×WÑWÓX×^Ñ^Ó`ÐdkÑk�Ü  ×!1Ñ!1Ð2FÑ!G×!YÑ!YÓZ×`Ñ`ÓbÐfoÑo�
Þžx®8¶zÜ$ð*óð ð
 %)§O¡O×$IÑ$IØ%¸Vó%Ð!ð ×#Ñ#Ð$8Ô9Ø$Ð$øð ×#Ñ#Ð$8Õ9ús   ÂE	G/ Ç/HT)Úexist_okÚ
state_dict)r¬   rO   Úsave_embedding_layersÚdefaultr/   ÚformatÚpt©ÚmetadataÚname_or_path)Úis_prompt_tuning)Úbase_model_classÚparent_libraryé   gÍ;fž ö?)Úauto_mapping_dict)1r™   rš   Úisfiler•   rs   rd   r   r–   ÚmakedirsÚcreate_or_update_model_cardr>   ÚgetÚjoinÚcollectionsÚdefaultdictÚitemsrr   ÚtorchÚTensorr?   ÚappendÚidÚlenÚcloneÚcopyr   r‡   Úsave_pretrainedÚis_contiguousÚ
contiguousÚsafe_save_filer5   Úsaver7   Úbase_model_name_or_pathrY   r\   Ú__dict__rc   Úinference_modeÚ	task_typeÚ_get_base_model_classÚ
__module__Ú__name__Úrr’   Ú
lora_alphar“   r”   )rb   Úsave_directoryÚsafe_serializationÚselected_adaptersr­   Úis_main_processr    r¢   r©   rO   rd   r¡   Ú
output_dirÚptrsÚnameÚtensorÚptrÚnamesÚshared_ptrsÚ_Úshared_tensor_nameÚkÚvrÏ   rµ   r¶   r¸   ÚkeyÚvals   `                            rh   rÈ   ÚPeftModel.save_pretrained¾   sÝ  ø€ ôT �7‰7�>‰>˜.×)Ñ)Ü˜¨~Ð.>Ð>aÐbÓcÐcàÑ$Ü $ T×%5Ñ%5×%:Ñ%:Ó%<Ó =Ñäô á->ó÷ ñ ô !ðÜ˜T×-Ñ-×2Ñ2Ó4Ó5Ð6°gÐ>OÐ=PÐPQðSóð õ
&	%öP Ü�KŠK˜°Ò6Ø×,Ñ,¨^Ô<ä-ˆLØ×*Ñ*¨<Ñ8ˆKä 9ØØ!Ÿ:™: l°DÓ9Ø)Ø&;ñ	!Ðð HTÐW`ÓG`œŸ™Ÿ™ nÔCÐftˆJÜ�KŠK˜
¨TÒ2ç×#5ô #×.Ò.¬tÓ4�Ø$5×$;Ñ$;Ö$=‘L�Dô " &¬%¯,©,×7Ñ7ØÔ.¨vÓ6Ñ7×>Ñ>¸tÖDð œR ›ZÑ(×/Ñ/°Ö5ñ %>ð =A¿J¹J¼LÔ[ºL©j¨c°5ÌCÐPUËJÐYZÉN›z˜s Ešz¹L�Ñ[à +× 1Ñ 1Ö 3‘H�A�uð /4°A°B«iÐ*Ø@QÐRdÑ@e×@kÑ@kÓ@mÐ)Ð*<Ó=ó /8ñ !4ð
 <ÑGÜ"&§-¢-°Ó"<�KØ48�KÔ1Ø×/Ñ/Ð0XÔYÙ(<Ø#ÐO`ó)Ð%ð .×3Ñ3Ö5‘D�A�qØŸ?™?×,Ó,Ø/0¯|©|«~Ð)¨!Ó,ñ 6ô Ø%Ü—G‘G—L‘L Ô-EÓFØ&¨Ð-óö
 !Ø;ÑGÜ"&§-¢-°Ó"<�KØ48�KÔ1Ø×/Ñ/Ð0XÔYÙ(<Ø#ÐO`ó)Ð%ô —
’
Ð,¬b¯g©g¯l©l¸:Ä|Ó.TÔUð ×2Ñ2Ñ:ð #×5×5ð —O‘O×,Ñ,×0Ñ0°ÀÔFàŸ™×.Ñ.×7Ñ7×;Ñ;¸NÈDÓQð Ô3ð
 )×7Ñ7ˆNØ)-ˆKÔ&à×$Ñ$Ñ,à#'×#=Ñ#=Ø%0×%CÑ%Cð $>ð $Ð ð "2×!<Ñ!<�ð )9×(AÑ(AØ&4ñ%Ñ!ð
 %)Ð!æØ;ÑGØ48�KÔ1Ø—M’M QÑ&•MØ&×1×1Ø#×.Ò.°!Ñ3Ö.ð $×.Ò.°&Ñ8Õ.à"×/×/ØQ\×QiÑQi×QoÑQoÔQqÔ3rÒQqÁXÀSÈ#°C¸¸S¹²LÑQqÒ3r˜Ô0Ø"×0×0ØR]×RkÑRk×RqÑRqÔRsÔ4tÒRsÁhÀcÈ3°S¸!¸c¹'²\ÑRsÒ4t˜Ô1à×+Ñ+¨JÐJ[Ð+Ñ\Ø)7ˆK×&òS .ùó6 \ùóR 4sùã4ts   Ç:U%ÈU%Ó"U+Ô,U1c
           
     óV  • SSK Jn  SSKJnJn  Uc”  U
R                  SS5      U
R                  SS5      U
R                  SS5      U
R                  SS5      S	.nU
R                  S
S5      =n(       a  XþS
'   [        [        R                  " U40 UD6   R                  " U40 U
D6nO:[        U[        5      (       a  U(       + Ul        O[        SUR                   35      eU	c  UR                  (       d  [        US0 5      n	[!        US5      (       a  XuR"                  l        OU(       a  [&        R(                  " S5        [!        US5      (       GaÏ  [+        [-        USS95      n[/        5       nSnUR1                  5        GH  u  nn[!        US5      (       d  M  [!        UR2                  S5      (       d  M7  [!        UR2                  R4                  S5      (       a*  UR2                  R4                  R6                  R8                  nUR2                  R:                  R=                  5        H_  nUR2                  R:                  U   [>        R@                  " S5      :X  d  M6  URC                  [E        U5      S-   [E        U5      -   5        Ma     GM     U(       a"  U
R                  SS5      (       d  [        S5      eU(       a^  UR=                  5        Vs0 s H>  nUU;   d  M  UUU   S   U[E        UU   RF                  5      RI                  SS5      S._M@     nnUU
S'   [        USS5      bL  [K        [/        URL                  RO                  5       5      RQ                  SS15      5      S :”  a  [S        U5        UR                  (       a  U(       a  [        S!5      eU(       + Ul        [        [        US"S5      U5      (       aò  [        X\5      (       d  [U        S#[W        U5       S$35      eS%U
;   a  U
S%   Ul,        O¹[Z        R\                  R_                  U5      (       d„  [a        5       nURc                  U5       Vs/ s H"  nUS&   S':X  d  M  US(   [K        U5      S-   S PM$     nn0 nU H%  n[Z        R\                  Re                  X"5      UU'   M'     UUl,        UUl3        OS%U
;  a  [        S)5      eURh                  UR=                  5       ;  a  U " UUUUUS*9nOXµRh                     " UUUUUS*9nURj                  " UU4UUUU	S+.U
D6nS, nURl                   Vs/ s H  nU" U5      (       d  M  UPM     nnU(       aU  S-U S3n [n        R
                  " URp                  5      n!U!(       a  UU!;   a  S.U S/U! S03U -   n [&        R(                  " U 5        U$ s  snf s  snf s  snf )1u�  
Instantiate a PEFT model from a pretrained model and loaded PEFT weights.

Note that the passed `model` may be modified inplace.

Args:
    model ([`torch.nn.Module`]):
        The model to be adapted. For ðŸ¤— Transformers models, the model should be initialized with the
        [`~transformers.PreTrainedModel.from_pretrained`].
    model_id (`str` or `os.PathLike`):
        The name of the PEFT configuration to use. Can be either:
            - A string, the `model id` of a PEFT configuration hosted inside a model repo on the Hugging Face
              Hub.
            - A path to a directory containing a PEFT configuration file saved using the `save_pretrained`
              method (`./my_peft_config_directory/`).
    adapter_name (`str`, *optional*, defaults to `"default"`):
        The name of the adapter to be loaded. This is useful for loading multiple adapters.
    is_trainable (`bool`, *optional*, defaults to `False`):
        Whether the adapter should be trainable or not. If `False`, the adapter will be frozen and can only be
        used for inference.
    config ([`~peft.PeftConfig`], *optional*):
        The configuration object to use instead of an automatically loaded configuration. This configuration
        object is mutually exclusive with `model_id` and `kwargs`. This is useful when configuration is already
        loaded before calling `from_pretrained`.
    autocast_adapter_dtype (`bool`, *optional*, defaults to `True`):
        Whether to autocast the adapter dtype. Defaults to `True`. Right now, this will only cast adapter
        weights using float16 and bfloat16 to float32, as this is typically required for stable training, and
        only affect select PEFT tuners. If set to `False`, the dtypes will stay the same as those of the
        corresponding layer.
    ephemeral_gpu_offload (`bool`, *optional*):
        Whether to use ephemeral GPU offloading for partially loaded modules. Defaults to `False`. This is
        useful when parts of the model and/or components (such as adapters) are kept in CPU memory until they
        are needed. Rather than perform expensive operations on small data, the data is transferred to the GPU
        on-demand, the operation(s) performed, and the results moved back to CPU memory. This brings a slight
        momentary VRAM overhead but gives orders of magnitude speedup in certain cases.
    low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
        Create empty adapter weights on meta device before loading the saved weights. Useful to speed up the
        process.
    torch_device (`str`, *optional*, defaults to None):
        The device to load the adapter on. If `None`, the device will be inferred.
    key_mapping (dict, *optional*, defaults to None)
        Extra mapping of PEFT `state_dict` keys applied before loading the `state_dict`. When this mapping is
        applied, the PEFT-specific `"base_model.model"` prefix is removed beforehand and the adapter name (e.g.
        `"default"`) is not inserted yet. Only pass this argument if you know what you're doing.
    kwargs: (`optional`):
        Additional keyword arguments passed along to the specific PEFT configuration class.

r/   )Ú MODEL_TYPE_TO_PEFT_MODEL_MAPPING)ÚXLoraConfigÚ
XLoraModelNr‘   ÚrevisionÚ	cache_dirÚtoken)r‘   rì   rí   rî   Úuse_auth_tokenz+The input config must be a PeftConfig, got Ú_checkpoint_conversion_mappingÚruntime_configzCEphemeral GPU offloading is not supported for this model. Ignoring.Úhf_device_mapT)ÚrecurseÚ_hf_hookÚoriginal_devicesÚdatasetÚmetar„   Úuse_safetensorsz8Disk offloading currently only supported for safetensorsÚsafetensors_fileztorch.Ú )rù   Úweight_nameÚdtypeÚoffload_indexÚcpuÚdiskr   úRCannot set a prompt learning adapter to trainable when loading pretrained adapter.r\   zExpected 'XLoraConfig', got 'z
' instead.rv   ÚtypeÚ	directoryrÜ   zFIf model_id is a local path, then `adapters` must be passed in kwargs.)rP   rL   )Úis_trainablerP   rL   Úkey_mappingc                ó.   • SU ;   a  gSU ;   a  gSU ;   a  gg)NÚvblora_vector_bankFÚprompt_encoderú.tinylora_v.T© )rã   s    rh   Úis_expected_missing_keyÚ:PeftModel.from_pretrained.<locals>.is_expected_missing_keyT  s&   € Ø# qÓ(ØØ 1Ó$Øà Ó"ØØro   z9Found missing adapter keys while loading the checkpoint: úAdapter name 'ú)' should not be contained in the prefix 'z@'. This could be the potential reason for missing adapter keys. )9Úautoré   Útunersrê   rë   r¼   r2   r1   Ú_get_peft_typeÚfrom_pretrainedrr   rÏ   r•   rg   rY   r_   r^   rñ   Úephemeral_gpu_offloadr—   r˜   Údictr   ÚsetÚnamed_modulesrô   Úweights_maprö   Úindexrõ   r   rÁ   ÚdeviceÚaddrt   rü   ÚreplacerÅ   rò   ÚvaluesÚintersectionr   Ú	TypeErrorr  rv   r™   rš   Úexistsr   Úlsr½   Ú_subfoldersrÐ   rœ   Úmissing_keysr3   rW   )"re   rc   Úmodel_idrO   r  rR   rP   r  rL   r  r¢   ré   rê   rë   Ú	hf_kwargsrï   Ú
weight_mapÚdisk_modulesr  rÜ   Úmodulerå   Úprý   ÚsÚfilerI   Úadapter_pathsÚload_resultr
  rã   r!  Úwarn_messagerŠ   s"                                     rh   r  ÚPeftModel.from_pretrainedŽ  sf  € õ| 	;ß3ð ‰>à#ŸZ™Z¨°TÓ:Ø"ŸJ™J z°4Ó8Ø#ŸZ™Z¨°TÓ:ØŸ™ G¨TÓ2ñ	ˆIð "(§¡Ð,<¸dÓ!CÐCˆ~ÕCØ.<Ð*Ñ+Ü0´×1JÒ1JÈ8Ñ1aÐW`Ñ1aÑb×rÒrØñØ"ñ‰Fô ˜¤
×+Ñ+Ø(4Ô$4ˆFÕ!äÐJÈ6×K[ÑK[ÐJ\Ð]Ó^Ð^ð Ñ¨&×*C×*CÜ! %Ð)IÈ2ÓNˆKô �6Ð+×,Ñ,Ø:O×!Ñ!Õ7æ$Ü—’ÐcÔdä�5˜/×*Ò*ÜÔ2°5À$ÑGÓHˆJô ›5ˆLØˆEØ %× 3Ñ 3× 5‘��fÜ˜6 :×.Ó.´7¸6¿?¹?ÐL^×3_Ó3_Ü˜vŸ™×:Ñ:¸I×FÑFØ &§¡× ;Ñ ;× CÑ C× IÑ I˜Ø%Ÿ™×?Ñ?×DÑDÖF˜Ø!Ÿ?™?×;Ñ;¸CÑ@ÄEÇLÂLÐQWÓDXÕXØ(×,Ñ,¬S°«Y¸©_¼sÀ3»xÑ-GÖHô  Gñ	 !6ö  F§J¡JÐ/@À$×$GÑ$GÜ Ð![Ó\Ð\æð (Ÿ_™_Ô.ó!ò /˜Ø˜LÑ(ó�AØ,1°!©HÐ5GÑ,HØ'(Ü!$ Z°¡]×%8Ñ%8Ó!9×!AÑ!AÀ(ÈBÓ!Oñò ñ
 /ð ð !ð +8��Ñ'ä�E˜?¨DÓ1Ñ=Ä3Ü�×#Ñ#×*Ñ*Ó,Ó-×:Ñ:¸EÀ6¸?ÓKóD
àóDô (¨Ô.à×$×$®ÜÐqÓrÐrà(4Ô$4ˆFÔ!Ü”g˜e \°4Ó8¸*×EÑEÜ˜f×2Ñ2ÜÐ"?ÄÀVÃ¸~ÈZÐ XÓYÐYØ˜VÓ#Ø"(¨Ñ"4�•ô —w‘w—~‘~ h×/Ñ/Ü$›�Að GHÇdÁdÈ8Änó%ÚFT¸dÐX\Ð]cÑXdÐhsÑXsÓ9˜˜V™¤S¨£]°QÑ%6Ð%8Ó9Ánð "ð %ð %'�MÛ(5˜Ü68·g±g·l±lÀ8Ó6V˜ lÓ3ñ )6à&3�F”OØ)6�FÕ&à!¨Ó/Ü(Ð)qÓrÐrà×ÑÐ#C×#HÑ#HÓ#JÓJÙØØØØ'=Ø"3ñ‰Eð 5×5EÑ5EÒFØØØØ'=Ø"3ñˆEð ×(Ò(ØØð
ð &Ø#9Ø/Ø#ñ
ð ñ
ˆò	ð $/×#;Ò#;ÓZÒ#;˜aÑ?VÐWX×?YŸÑ#;ˆÐZÞð WÐWcÐVdÐdeÐfˆLä0×4Ò4°V×5EÑ5EÓFˆFÞ˜,¨&Ó0à$ \ NÐ2[Ð\bÐ[cð dTð Tà ñ !�ô
 �MŠM˜,Ô'àˆùòQ!ùò@%ùòl [s$   Ë
VË#7VÑ V!ÑV!Ô!V&Ô6V&c                ó4  • U R                   U   n[        U S5      (       d+  [        R                  R	                  0 5      U l        0 U l        S nU R                  R                  5        HG  u  pEUR                  5        H
  nSUl
        M     [        U[        5      (       d  M:  Ub  M?  UnX@l        MI     Uc  U R                  nUR                  c'  UR                  [         R"                  :X  a  SOSUl        S n U R                  R%                  S5      nUGc-  [)        UR+                  5       5       GH  u  p‰[-        U	SS 5      n
[        U R                  R.                  S5      (       a/  U R                  R.                  R1                  5       R2                  nOeSU R                  R.                  ;   a+  U R                  R.                  R4                  R2                  nO U R                  R.                  R2                  nU	R6                  S	   U:X  d  U
c  Mä  U
S	   U:X  d  Mï  UR%                  UR9                  S
S5      5      n  O   Xpl        [<        UR>                     nUR>                  [@        RB                  [@        RD                  [@        RF                  4;   a  U" X R:                  5      nO¾UR>                  [@        RH                  :X  a	  U" U5      nO—UR>                  [@        RJ                  [@        RL                  4;   a^  [O        S U RQ                  5       RS                  5        5       5      (       a"  [U        UR>                  RV                   S35      eU" U5      nO[U        S5      eURY                  U RZ                  5      nU R
                  R]                  [        R                  R	                  X05      5        [        R^                  " UR`                  UR                  -  5      Rc                  5       U R                  U'   g ! [&         a     GNf = f)Nr  Fr·   r/   zembeddings.word_embeddingsÚds_shapeÚget_text_configÚtext_configr   z.weightrú   c              3  ó<   #   • U  H  n[        US S5      v •  M     g7f)Úgradient_checkpointingFN©r_   )r€   r&  s     rh   r‚   Ú2PeftModel._setup_prompt_encoder.<locals>.<genexpr>²  s   é € ÐrÒRqÈ”7˜6Ð#;¸U×CÐCÒRqùs   ‚z+ does not work with gradient checkpointing.zNot supported)2rd   r^   rÁ   ÚnnÚ
ModuleDictr  Úprompt_tokensr\   Únamed_childrenÚ
parametersÚrequires_gradrr   r    Útransformer_backbone_nameÚnum_transformer_submodulesrÐ   r9   ÚSEQ_2_SEQ_LMÚget_submoduleru   rs   Únamed_parametersr_   rR   r0  Ú
vocab_sizer1  Úshaper  Úword_embeddingsr4   rW   r8   ÚPROMPT_TUNINGÚMULTITASK_PROMPT_TUNINGÚCPTÚP_TUNINGÚPREFIX_TUNINGÚ	CARTRIDGEr–   Úget_base_modelÚmodulesr•   r|   Útor  ÚupdateÚarangeÚnum_virtual_tokensÚlong)rb   rO   rR   Útransformer_backbonerÜ   r&  ÚparamrC  Únamed_paramr|   Ú"deepspeed_distributed_tensor_shaperA  Ú	model_clsr  s                 rh   Ú_setup_prompt_encoderÚPeftModel._setup_prompt_encoderr  s[  € Ø×!Ñ! ,Ñ/ˆÜ�tÐ-×.Ñ.Ü"'§(¡(×"5Ñ"5°bÓ"9ˆDÔØ!#ˆDÔØ#ÐØ ŸO™O×:Ñ:Ö<‰LˆDØ×*Ñ*Ö,�Ø&+�Ö#ñ -ä˜&¤/×2Ó2à'Ó/Ø+1Ð(Ø59Ö2ñ =ð  Ñ'Ø#'§?¡?Ð à×,Ñ,Ñ4Ø5;×5EÑ5EÌ×I^ÑI^Ó5^±ÐdeˆFÔ-ð ˆð	ð #Ÿo™o×;Ñ;Ð<XÓYˆOð Ò"ô '+Ð+?×+PÑ+PÓ+R×&SÑ"�ô
 6=¸UÀJÐPTÓ5UÐ2ô ˜4Ÿ?™?×1Ñ1Ð3D×EÑEØ!%§¡×!7Ñ!7×!GÑ!GÓ!I×!TÑ!T‘Jà" d§o¡o×&<Ñ&<Ó<Ø!%§¡×!7Ñ!7×!CÑ!C×!NÑ!N‘Jà!%§¡×!7Ñ!7×!BÑ!B�Jà—;‘;˜q‘> ZÓ/Ø6ÓBØ:¸1Ñ=ÀÕKà&:×&HÑ&HÈ×I\ÑI\Ð]fÐhjÓIkÓ&l�OÙñ+ 'Tð.  /ÔÜ.¨v×/?Ñ/?Ñ@ˆ	à×Ñ¤× 6Ñ 6¼×8XÑ8XÔZb×ZfÑZfÐgÓgÙ& v×/CÑ/CÓD‰NØ×Ñ¤×!2Ñ!2Ó2Ù& vÓ.‰NØ×Ñ¤(×"8Ñ"8¼(×:LÑ:LÐ!MÓMäÑrÐRV×ReÑReÓRg×RoÑRoÔRqÓr×rÑrÜ  F×$4Ñ$4×$:Ñ$:Ð#;Ð;fÐ!gÓhÐhÙ& vÓ.‰Nä˜_Ó-Ð-à'×*Ñ*¨4¯;©;Ó7ˆØ×Ñ×"Ñ"¤5§8¡8×#6Ñ#6¸Ð7UÓ#VÔWÜ+0¯<ª<Ø×%Ñ%¨×(IÑ(IÑIó,
ç
‰$‹&ð 	×Ñ˜<Ò(øô] ó 	Úð	ús   Ã9P	 Ð	
PÐPc                ó&   • U R                  U5        g)z<
Prepares the model for gradient checkpointing if necessary
N)Ú)_prepare_model_for_gradient_checkpointing)rb   rc   s     rh   r`   Ú2PeftModel.prepare_model_for_gradient_checkpointing¾  s   € ð 	×6Ñ6°uÕ=ro   c                ó  • [        USS5      (       dz  [        USS5      (       dh  [        USS5      (       dV  [        US5      (       a  UR                  5         U$ [        US5      (       a"  S nUR                  5       R	                  U5        U$ )NÚis_loaded_in_8bitFÚis_loaded_in_4bitÚis_quantizedÚenable_input_require_gradsÚget_input_embeddingsc                ó&   • UR                  S5        g )NT)Úrequires_grad_)r&  ÚinputÚoutputs      rh   Úmake_inputs_require_gradÚUPeftModel._prepare_model_for_gradient_checkpointing.<locals>.make_inputs_require_gradÎ  s   € Ø×)Ñ)¨$Õ/ro   )r_   r^   r_  r`  Úregister_forward_hook)rb   rc   re  s      rh   rY  Ú3PeftModel._prepare_model_for_gradient_checkpointingÄ  s†   € ä�EÐ.°×6Ñ6Ü�uÐ1°5×9Ñ9Ü�u˜n¨e×4Ñ4ä�uÐ:×;Ñ;Ø×0Ñ0Ô2ð ˆô ˜Ð 6×7Ñ7ò0ð ×*Ñ*Ó,×BÑBÐC[Ô\Øˆro   c                óœ  >• U R                   U   nU R                  U   R                  S5      R                  SS5      R	                  UR
                  R                  R                  5      nU R                  U   R                  nU R                  U   R                  [        R                  [        R                  4;   a"  USS2SU R                  U   R                  24   nU R                  U   R                  [        R                  :X  a  [        U   n[         XR]G  U5      nOU" U5      nUS   R%                  5       R'                  5       $ )zr
Returns the prompt embedding to save when saving the model. Only applicable when using a prompt learning
method.
r   r/   éÿÿÿÿN)r  r8  Ú	unsqueezeÚexpandrL  Ú	embeddingÚweightr  rd   rW   r8   rH  rI  rO  rE  r4   rT   ÚforwardÚdetachrþ   )rb   rO   r  r8  rW   Úprompt_embedding_clsÚprompt_embeddingsrg   s          €rh   Úget_prompt_embedding_to_saveÚ&PeftModel.get_prompt_embedding_to_saveÔ  s*  ø€ ð
 ×,Ñ,¨\Ñ:ˆà×Ñ˜|Ñ,×6Ñ6°qÓ9×@Ñ@ÀÀBÓG×JÑJÈ>×KcÑKc×KjÑKj×KqÑKqÓrð 	ð ×$Ñ$ \Ñ2×<Ñ<ˆ	Ø×Ñ˜LÑ)×3Ñ3¼×8NÑ8NÔPX×PbÑPbÐ7cÓcØ)ª!Ð-`¨t×/?Ñ/?ÀÑ/M×/`Ñ/`Ð-`Ð*`ÑaˆMà×Ñ˜LÑ)×3Ñ3´x×7WÑ7WÓWÜ#=¸iÑ#HÐ Ü %Ð&:Ñ SÐTaÓ bÑá .¨}Ó =Ðà  Ñ#×*Ñ*Ó,×0Ñ0Ó2Ð2ro   c                ó4  • U R                   nU R                  U R                     nU R                  U R                     R	                  S5      R                  US5      R                  UR                  R                  R                  5      nUR                  [        R                  [        R                  4;   Gan  USS2SUR                  24   nUR                  (       a(  UR                  R                  R!                  USS5      nOU" U5      nU R"                  b  UR                  U R"                  5      nUR%                  UUR                  UR&                  S-  UR(                  UR*                  UR(                  -  5      nUR,                  S:X  a  [.        R0                  " Xw/SS9nUR3                  / SQ5      R5                  UR,                  S-  5      nU R7                  5       n[9        USS5      n	[9        U	S	S
5      n
[:        R<                  " U R>                  R@                  S5      b'  [:        U R>                  R@                     nU" U5      nGOµSU
;   d  SU
;   Ga@  [B        RD                  RG                  [H        RJ                  5      [B        RD                  RG                  S5      :  nU(       a  Ub  US:X  a  [M        S5      eUR>                  n[O        US5      (       a  URQ                  5       nU(       a+  SSK$J)n  U" UUUUS   RT                  US   R                  S9nO	[W        US9n[.        RX                  " UR                  US   R                  S9n[[        UR&                  5       H'  nUS   U   US   U   nnUR]                  UUUSU0S9  M)     UnGOhUR,                  S:X  aw  [B        RD                  RG                  [H        RJ                  5      [B        RD                  RG                  S5      :  nU(       a  [V        R^                  " U5      nOí[W        U5      nOáUR,                  S:X  aÑ  [9        U R`                  SS5      (       aµ  [B        RD                  RG                  [H        RJ                  5      [B        RD                  RG                  S5      :  nU(       a  [b        R^                  " U5      nO[c        U5      n[W        5       Ul2        URf                  Ri                  5        H  nSURf                  U'   M     [k        U R7                  5       U5        U$ UR                  [        Rl                  :X  a
  U" Xb5      nU$ UR                  (       a  UR                  R                  nOUSS nU" U5      nUR!                  USS5      nU$ )zc
Returns the virtual prompts to use for Peft. Only applicable when using a prompt learning method.
r   rj  Nr/   r·   ©Údim)r·   r   é   r/   é   rR   Ú
model_typerú   Úgemma2Úgemma3_textz4.56.0.dev0z«max_cache_len is missing but it should have been passed. Something went wrong, please open an issue on GitHub with a reproducer: https://github.com/huggingface/peft/issuesr0  )ÚHybridCache)rR   Úmax_batch_sizeÚmax_cache_lenrü   r  )rR   ©r  Úcache_position)Úcache_kwargsÚ_supports_cache_classTF)7Úactive_peft_configr  rV   r8  rk  rl  rL  rm  rn  r  rW   r8   rH  rI  rO  rÏ   ÚrepeatÚbase_model_torch_dtypeÚviewÚ
num_layersÚnum_attention_headsÚ	token_dimr=  rÁ   ÚcatÚpermuteÚsplitrJ  r_   r6   r¼   rR   rz  Ú	packagingÚversionÚparseÚtransformersr0   r•   r^   r0  r}  rü   r   rN  ÚrangerM  Úfrom_legacy_cacher\   r   Úcross_attention_cacheÚ
is_updatedr   rB   rE  )rb   Ú
batch_sizeÚtask_idsr  rd   r  r8  Úpast_key_valuesr\   Úmodel_configrz  Úpost_process_fnÚtransformers_lt_4_56Úbase_configr}  Ú	new_cacher�  Ú	layer_idxÚ
key_statesÚvalue_statesrå   Úpromptss                         rh   Ú
get_promptÚPeftModel.get_prompté  s  € ð ×-Ñ-ˆØ×,Ñ,¨T×-@Ñ-@ÑAˆà×Ñ˜t×2Ñ2Ñ3ß‰Y�q‹\ß‰V�J Ó#ß‰R�×(Ñ(×/Ñ/×6Ñ6Ó7ð	 	ð × Ñ ¤X×%;Ñ%;¼X×=OÑ=OÐ$PÔPØ)ª!Ð-M¨{×/MÑ/MÐ-MÐ*MÑNˆMØ×)×)Ø"0×":Ñ":×"AÑ"A×"HÑ"HÈÐUVÐXYÓ"Z‘á"0°Ó"?�Ø×*Ñ*Ñ6Ø"1×"4Ñ"4°T×5PÑ5PÓ"Q�Ø-×2Ñ2ØØ×.Ñ.Ø×&Ñ&¨Ñ*Ø×/Ñ/Ø×%Ñ%¨×)HÑ)HÑHóˆOð ×5Ñ5¸Ó:Ü"'§)¢)¨_Ð,NÐTUÑ"V�ð .×5Ñ5²oÓF×LÑLØ×6Ñ6¸Ñ:óˆOð ×,Ñ,Ó.ˆJÜ" :¨x¸Ó>ˆLÜ  ¨|¸RÓ@ˆJÜG×KÒKÈDÏKÉK×LbÑLbÐdhÓiÑuÜ"ZÐ[_×[fÑ[f×[qÑ[qÑ"r�Ù"1°/Ó"B’Ø˜jÓ(¨m¸zÔ.Iä'0×'8Ñ'8×'>Ñ'>¼|×?WÑ?WÓ'XÔ[d×[lÑ[l×[rÑ[rØ!ó\ñ (Ð$ö (¨mÑ.CÈÐZ\ÓI\Ü$ðhóð ð )×/Ñ/�Ü˜;Ð(9×:Ñ:Ø"-×"=Ñ"=Ó"?�KÞ'õ 9á +Ø*Ø'1Ø&3Ø-¨aÑ0×6Ñ6Ø.¨qÑ1×8Ñ8ñ!‘Iô !-°KÑ @�IÜ!&§¢¨k×.LÑ.LÐUdÐefÑUg×UnÑUnÑ!o�Ü!& {×'=Ñ'=Ö!>�IØ/>¸qÑ/AÀ)Ñ/LÈoÐ^_ÑN`ÐajÑNk �JØ×$Ñ$Ø" L°)ÐK[Ð]kÐJlð %ó ñ "?ð
 #,’Ø×7Ñ7¸1Ó<ô (1×'8Ñ'8×'>Ñ'>¼|×?WÑ?WÓ'XÔ[d×[lÑ[l×[rÑ[rØ!ó\ñ (Ð$ö (Ü&2×&DÒ&DÀ_Ó&U‘Oä&2°?Ó&C‘Oà×8Ñ8¸AÓ=Ä7Ø—‘Ð!8¸$÷Dñ Dô (1×'8Ñ'8×'>Ñ'>¼|×?WÑ?WÓ'XÔ[d×[lÑ[l×[rÑ[rØ!ó\ñ (Ð$ö (Ü&9×&KÒ&KÈOÓ&\‘Oä&9¸/Ó&J�Oä8D»�Ô5à*×5Ñ5×:Ñ:Ö<�CØ6;�O×.Ñ.¨sÓ3ñ =ä)¨$×*=Ñ*=Ó*?ÀÔQØ"Ð"à×$Ñ$¬×(HÑ(HÓHÙ(¨ÓA�ð ˆNð ×-×-Ø,×6Ñ6×=Ñ=‘Gð %2°"°1Ð$5�MÙ,¨]Ó;�GØ!Ÿ.™.¨°Q¸Ó:�ØˆNro   c                óº  • SnSnU R                  5        HÁ  u  p4UR                  5       nUS:X  a  [        US5      (       a  UR                  nUR                  R
                  S:X  aT  [        US5      (       a  UR                  5       nO*[        US5      (       d  SnOUR                  R                  nUS-  U-  nX%-  nUR                  (       d  M½  X-  nMÃ     X4$ )z[
Returns the number of trainable parameters and the number of all parameters in the model.
r   Úds_numelÚ
Params4bitÚelement_sizeÚquant_storager/   r·   )
r@  Únumelr^   r¥  rg   rÓ   r§  r¨  Úitemsizer;  )rb   Útrainable_paramsÚ	all_paramrá   rR  Ú
num_paramsÚ	num_bytess          rh   Úget_nb_trainable_parametersÚ%PeftModel.get_nb_trainable_parametersf  sÓ   € ð ÐØˆ	Ø×-Ñ-Ö/‰HˆAØŸ™›ˆJà˜Q‹¤7¨5°*×#=Ñ#=Ø"Ÿ^™^�
ð
 �‰×'Ñ'¨<Ó7Ü˜5 .×1Ñ1Ø %× 2Ñ 2Ó 4‘IÜ  ¨×8Ñ8Ø !‘Ià %× 3Ñ 3× <Ñ <�IØ'¨!™^¨iÑ7�
àÑ#ˆIØ×"×"Ñ"Ø Ñ.Ò ñ) 0ð,  Ð*Ð*ro   c           	     ób   • U R                  5       u  p[        SUS SUS SSU-  U-  S 35        g)až  
Prints the number of trainable parameters in the model.

Note: print_trainable_parameters() uses get_nb_trainable_parameters() which is different from
num_parameters(only_trainable=True) from huggingface/transformers. get_nb_trainable_parameters() returns
(trainable parameters, all parameters) of the Peft Model which includes modified backbone transformer model.
For techniques like LoRA, the backbone transformer model is modified in place with LoRA modules. However, for
prompt tuning, the backbone transformer model is unmodified. num_parameters(only_trainable=True) returns number
of trainable parameters of the backbone transformer model which can be different.
ztrainable params: z,dz || all params: z || trainable%: éd   z.4fN)r¯  Úprint)rb   r«  r¬  s      rh   Úprint_trainable_parametersÚ$PeftModel.print_trainable_parameters„  sa   € ð '+×&FÑ&FÓ&HÑ#ÐäØ Ð!1°"Ð 5Ð5EÀiÐPRÀ^ÐScÐdgÐjzÑdzð  ~Gñ  eGð  HKð  dLð  Mõ	
ro   c                ó~   >•  [         TU ]  U5      $ ! [         a     US:X  a  e [        U R                  U5      s $ f = f)z1Forward missing attributes to the wrapped module.r\   )rT   Ú__getattr__ru   r_   r\   )rb   rÜ   rg   s     €rh   r·  ÚPeftModel.__getattr__•  sC   ø€ ð	2Ü‘7Ñ& tÓ,Ð,øÜó 	2Ø�|Ó#ØÜ˜4Ÿ?™?¨DÓ1Ò1ð	2ús   ƒ ’'<»<c              /  óâ   #   • [        U R                  S5      (       a;  U R                  (       a*  U R                  R                  " U0 UD6   S v •  S S S 5        g S v •  g ! , (       d  f       g = f7f)NÚ_enable_peft_forward_hooks)r^   r\   ry   rº  )rb   Úargsr¢   s      rh   rº  Ú$PeftModel._enable_peft_forward_hooksž  sX   é € ô �4—?‘?Ð$@×AÑAÀd×Fe×FeØ—‘×;Ò;¸TÐLÀVÓLÛ÷ Màó Ø÷ MÔLàüs   ‚A	A/ÁAÁA/Á
A,Á(A/c                óþ   • U R                   " U0 UD6   UR                  5        VVs0 s H  u  p4X0R                  ;  d  M  X4_M     nnnU R                  5       " U0 UD6sSSS5        $ s  snnf ! , (       d  f       g= f)z
Forward pass of the model.
N)rº  rÀ   rX   rJ  ©rb   r»  r¢   rã   rä   s        rh   ro  ÚPeftModel.forward«  sl   € ð ×,Ò,¨dÐ=°fÓ=Ø'-§|¡|¤~Ôa¢~™t˜q¸×B`ÑB`Ñ9`“d�a’d¡~ˆFÑaØ×&Ñ&Ô(¨$Ð9°&Ñ9÷ >Ñ=ùÛa÷ >Õ=ús"   “A.§A(Á A(ÁA.Á(A.Á.
A<c                ó  • U R                   " U0 UD6   UR                  5        VVs0 s H  u  p4X0R                  ;  d  M  X4_M     nnnU R                  5       R                  " U0 UD6sS S S 5        $ s  snnf ! , (       d  f       g = frk   )rº  rÀ   rX   rJ  Úgenerater¾  s        rh   rÁ  ÚPeftModel.generate³  sp   € Ø×,Ò,¨dÐ=°fÓ=Ø'-§|¡|¤~Ôa¢~™t˜q¸×B`ÑB`Ñ9`“d�a’d¡~ˆFÑaØ×&Ñ&Ó(×1Ò1°4ÐB¸6ÑB÷ >Ñ=ùÛa÷ >Õ=ús"   “A8§A2Á A2Á"A8Á2A8Á8
Bc                ó|   • U(       d   U R                   R                  R                  $ U R                   R                  $ )z
Returns the base model class.
)r\   rc   rg   )rb   r´   s     rh   rÑ   ÚPeftModel._get_base_model_class¸  s-   € ö  Ø—?‘?×(Ñ(×2Ñ2Ð2Ø�‰×(Ñ(Ð(ro   c              #  ó  #   • U R                   U R                     R                  (       an   U R                  nU R                  R                  U l        U R
                  nU R                  R
                  U l        SU l        Sv •  Xl        X l        SU l        gU R                   U R                     R                  (       aH   U R                  R                  5         SU l        Sv •  U R                  R                  5         SU l        gU R                  5       nUR                  S:X  a  [        R                  " S5         U R                  R                  5         SU l        Sv •  UR                  SLa  U R                  R                  5         SU l        g! WU l        WU l        SU l        f = f! U R                  R                  5         SU l        f = f! UR                  SLa  U R                  R                  5         SU l        f = f7f)z­
Context manager that disables the adapter module. Use this to run inference on the base model.

Example:

```py
>>> with model.disable_adapter():
...     model(inputs)
```
TNFÚ	irregularzÀThe model contains some adapter layers that are enabled and others that are disabled. This is most likely unintentional. After exiting the disable_adapter context, all adapters will be enabled)rd   rV   rY   ro  r\   Úprepare_inputs_for_generationra   Úis_adaption_promptÚdisable_adapter_layersÚenable_adapter_layersÚget_model_statusÚenabledr—   r˜   )rb   Úold_forwardÚ!old_prepare_inputs_for_generationÚmodel_statuss       rh   Údisable_adapterÚPeftModel.disable_adapterÀ  s£  é € ð ×Ñ˜D×/Ñ/Ñ0×C×Cð0ð #Ÿl™l�Ø#Ÿ™×6Ñ6�”Ø48×4VÑ4VÐ1Ø59·_±_×5bÑ5b�Ô2Ø*.�Ô'Ûà*”Ø5VÔ2Ø*/�Õ'à×Ñ˜d×1Ñ1Ñ2×E×Eð0Ø—‘×6Ñ6Ô8Ø*.�Ô'Ûà—‘×5Ñ5Ô7Ø*/�Õ'ð  ×0Ñ0Ó2ˆLØ×#Ñ# {Ó2Ü—’ð&ôð
0Ø—‘×6Ñ6Ô8Ø*.�Ô'Ûà×'Ñ'¨uÒ4à—O‘O×9Ñ9Ô;Ø*/�Õ'øð;  +�”Ø5V�Ô2Ø*/�Õ'ûð —‘×5Ñ5Ô7Ø*/�Õ'ûð  ×'Ñ'¨uÒ4à—O‘O×9Ñ9Ô;Ø*/�Õ'üsN   ‚)H¬AF Â<HÃ%F0 Ã'AHÅ %G Å%1HÆF-Æ-HÆ0#GÇHÇ2HÈHc                ó|   • U R                   R                  (       a  U R                  $ U R                  R                  $ )z
Returns the base model.
)r„  rY   r\   rc   rm   s    rh   rJ  ÚPeftModel.get_base_modelö  s.   € ð #'×"9Ñ"9×"L×"Lˆt�‰ÐgÐRV×RaÑRa×RgÑRgÐgro   c                óà  • [         R                  " UR                  5      nU(       a"  X;   a  [        R                  " SU SU S35        UR                  U R                  :w  a&  [        SU R                   SUR                   S35      e UR                  (       a_  X R                  U'   [        X R                  5      nU R                  U5        [        U R                  U[        R                  " U 5      US9  O”UR                  (       aF  U R                  R!                  X5        [        U R                  U[        R                  " U 5      US9  O=X R                  U'   U R                  R#                  U R                  R$                  XS9   [)        U R                  S	5      (       a  U R                  R+                  XS
9  gg! [&         a    XR                  ;   a  U R                  U	 e f = f)á‘  
Add an adapter to the model based on the passed configuration.

This adapter is not trained. To load a trained adapter, check out [`PeftModel.load_adapter`].

The name for the new adapter should be unique.

The new adapter is not automatically set as the active adapter. Use [`PeftModel.set_adapter`] to set the active
adapter.

Args:
    adapter_name (`str`):
        The name of the adapter to be added.
    peft_config ([`PeftConfig`]):
        The configuration of the adapter to be added.
    low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
        Create empty adapter weights on meta device. Useful to speed up the process when loading saved
        adapters. Don't use this option when creating a new PEFT adapter for training.
    autocast_adapter_dtype (`bool`, *optional*, defaults to `True`):
        Whether to autocast the adapter dtype. Defaults to `True`. Right now, this will only cast adapter
        weights using float16 and bfloat16 to float32, as this is typically required for stable training, and
        only affect select PEFT tuners. If set to `False`, the dtypes will stay the same as those of the
        corresponding layer.
r  r  zK'. This may lead to reinitialization of the adapter weights during loading.z9Cannot combine adapters with different peft types. Found z and r„   )rc   rd   r™  rO   rK   rM   rN   N)r3   r¼   rW   r—   r˜   r•   rY   rd   r;   rR   rV  r.   r\   r'   Úget_model_configrÈ  r]   Úinject_adapterrc   Ú	Exceptionr^   rM   )rb   rO   rd   rL   rP   rŠ   s         rh   r]   ÚPeftModel.add_adapterü  sÂ  € ô> -×0Ò0°×1FÑ1FÓGˆÞ�lÓ,Ü�MŠMØ   Ð.WÐX^ÐW_ð `[ð [ôð
 × Ñ  D§N¡NÓ2ÜðØŸ™Ð(¨¨k×.CÑ.CÐ-DÀAðGóð ð
	Ø×-×-Ø1<× Ñ  Ñ.Ü=¸kÏ;É;ÓW�Ø×*Ñ*¨<Ô8Ü0ØŸ/™/Ø +Ü!*×!;Ò!;¸DÓ!AØ!-ó	ð ×/×/Ø—‘×+Ñ+¨LÔFÜ0ØŸ/™/Ø +Ü!*×!;Ò!;¸DÓ!AØ!-ó	ð 2=× Ñ  Ñ.Ø—‘×.Ñ.Ø—O‘O×)Ñ)¨<ð /ò ô �4—?‘?Ð$9×:Ñ:Ø�O‰O×/Ñ/Ø)ð 0ò ð ;øô ó 	Ø×/Ñ/Ó/Ø×$Ñ$ \Ð2Øð	ús   ÂA/G Ã;AG Å<G Ç(G-c                óÂ   • XR                   ;  a  [        SU S35      eU R                  R                  US9  U R                  n[        U5      nUS:X  a  US   U l        gg)z`
Deletes an existing adapter.

Args:
    adapter_name (str): Name of the adapter to be deleted.
úAdapter z does not exist)rO   r/   r   N)rd   r•   r\   rŸ   rq   rÅ   rV   )rb   rO   Únew_active_adaptersÚnum_adapterss       rh   rŸ   ÚPeftModel.delete_adapterJ  sl   € ð ×/Ñ/Ó/Ü˜x¨ ~°_ÐEÓFÐFà�‰×&Ñ&°LÐ&ÑAØ"×2Ñ2ÐÜÐ.Ó/ˆð ˜1ÓØ"5°aÑ"8ˆDÕð ro   c                óÂ   • [        5       nU R                  R                  5        H.  n[        USS 5      c  M  UR	                  UR
                  5        M0     U(       d  g U$ )NÚmodules_to_save)r  rd   r  r_   rM  rà  )rb   rK  rR   s      rh   rà  ÚPeftModel.modules_to_save]  sQ   € ä›EˆØ×&Ñ&×-Ñ-Ö/ˆFÜ�vÐ0°$Ó7ÓCà—‘˜v×5Ñ5Ö6ñ 0ö
 àØˆro   c                ó   • [        U 5      $ )aÀ  Get the status of each adapter layer in the model.

This method returns a list of `TunerLayerStatus` dataclass instances, each of which contains the following
attributes:

- `name` (`str`):
   The name of the adapter layer, e.g. `model.encoder.block.0.layer.0.SelfAttention.q`.
- `module_type` (`str`):
   The type of the adapter layer, e.g. `lora.Linear`.
- `enabled` (`bool`):
   Whether the adapter layer is enabled.
- `active_adapters` (`list[str]`):
   The names of the active adapters, if any, e.g. `["default"]`.
- `merged_adapters` (`list[str]`):
   The names of the merged adapters, if any, e.g. `["default"]`.
- `available_adapters` (`list[str]`):
   The names of the available adapters, e.g. `["default"]`.

Args:
    model ([`~PeftModel`]):
        The model to get the adapter layer status from.

Returns:
    list[`peft.peft_model.TunerLayerStatus`]:
        A list of dataclasses, each containing the status of the corresponding adapter layer.

)Úget_layer_statusrm   s    rh   rã  ÚPeftModel.get_layer_statusj  s   € ô8   Ó%Ð%ro   c                ó   • [        U 5      $ )aÍ  Get the status of tuners of the model.

This method returns a `TunerModelStatus` dataclass instance, which contains the following attributes:

- `base_model_type` (`str`):
   The type of the base model, e.g. `T5Model`.
- `adapter_model_type` (`str`):
   The type of the adapter model, e.g. `LoraModel`.
- `peft_types` (`dict[str, str]`):
   The mapping of adapter name to adapter type, e.g. `{"default": "LORA"}`.
- `trainable_params` (`int`):
   The number of trainable parameters in the model.
- `total_params` (`int`):
   The total number of parameters in the model.
- `num_adapter_layers` (`int`):
   The number of adapter layers in the model.
- `enabled` (`bool`, `Literal["irregular"]`):
   Whether all adapter layers are enabled. If some are enabled and some are not, this will be `"irregular"`.
   This means that your model is in an inconsistent state and might not work as expected.
- `active_adapters` (`list[str]`, `Literal["irregular"]`):
   The names of the active adapters. If the active adapters are not consistent across all layers, this will be
   `"irregular"`, which means that your model is in an inconsistent state and might not work as expected.
- `merged_adapters` (`list[str]`, `Literal["irregular"]`):
   The names of the merged adapters. If the merged adapters are not consistent across all layers, this will be
   `"irregular"`, which means that your model is in an inconsistent state and might not work as expected.
- `available_adapters` (`list[str]`):
   The names of the available adapters, e.g. `["default"]`.

Args:
    model ([`~PeftModel`]):
        The model to get the adapter layer status from.

Returns:
    `peft.peft_model.TunerModelStatus`:
        A dataclass containing the status of the model.

)rË  rm   s    rh   rË  ÚPeftModel.get_model_statusˆ  s   € ôL   Ó%Ð%ro   c                ó´   • Sn0 n0 nUR                  5        H<  u  pVU[        R                  " [        5      R                  ;   d  XR;   a  XcU'   M8  XdU'   M>     X44$ )N)rï   )rÀ   ÚinspectÚ	signaturer   r:  )re   r¢   Ú(_kwargs_not_in_hf_hub_download_signatureÚhf_hub_download_kwargsÚother_kwargsrå   r|   s          rh   Ú_split_kwargsÚPeftModel._split_kwargs°  s^   € à3FÐ0Ø!#ÐØˆà Ÿ,™,ž.‰JˆCØ”g×'Ò'¬Ó8×CÑCÓCÀsÓGvØ.3 sÓ+à$)˜SÓ!ñ	 )ð &Ð3Ð3ro   c           
     ó®  • U(       d  U$ Sn[        U R                  R                  5       5      nU GH  n[        UR                  5       5      nUS   R                  S5      S   S-   nU Ho  nUR	                  S5      n	X8SU	 -   n
[        U R                  5       5      U
   n[        U[        5      (       a  X8SU	 -   S-   X‰S -   nOX8-   nXÁU   S'   X   X'   X	 Mq     [        5       n[        UR                  5       5       GH3  nX   S   n[        UR                  [        R                  5      5      n[        U5       H  u  nnSU;   d  M  UU==   S	-  ss'     O   [        R                  R                  " U6 nXí;   a  M�  0 n[        US
S9 nUR                  5        Hö  nUR!                  U5      nUR#                  5       nUR	                  S5      n	X7-   USU	 -   n
[        U R                  5       5      U
   n[        U[        5      (       az  U
S-   UU	S -   nUR%                  5        VVs0 s H  u  nnX¨;   d  M  UU_M     nnnUR%                  5        H-  u  nnUR	                  S5      nUSU SU 3-   UUS -   nUUU'   M/     OX7-   U-   nUUU'   Mø     UR'                  U5        UR                  5        H
  nUUS.X'   M     [        R                  R)                  U5      n[        R                  R+                  U5      (       d  [        R,                  " U5        [/        UUWS9  SSS5        GM6     GM
     gs  snnf ! , (       d  f       GMT  = f)ab  
Update the offload_index and safetensors files for loading and mergine PeftModels with disk-offloaded modules.

Args:
    offload_index (Dict[str: str]):
        Dictionary of disk-offloaded modules with their metadata and safetensors filenames
    adapters_weights (Dict[str: torch.tensor]):
        Dictionary of Peft adapter module names and weights
zbase_model.model.r   r„   Nz.base_layerrû   rù   z--z-peftr°   )Ú	framework)rù   rû   r±   )rs   rd   r   r�  Úrfindr  r  rr   r(   r  r™   ÚsepÚ	enumeraterš   r½   r   Ú
get_tensorr²   rÀ   r  r�   r  rº   rË   ) rb   rý   Úadapters_weightsrŠ   rI   rO   r   Úblock_idrå   Ú
suffix_posÚextended_prefixr&  Únew_keyÚ
files_seenÚfnameÚnew_fname_listÚirÜ   Ú	new_fnameÚ	safe_dictÚfÚsafe_keyÚsafe_tensorr²   Úsafe_moduleÚ	final_keyræ   Ú	lora_dictÚlora_keyÚlora_valÚdivideÚ	base_names                                    rh   Ú_update_offloadÚPeftModel._update_offload¾  sF  € ö Ø Ð à$ˆä˜T×-Ñ-×2Ñ2Ó4Ó5ˆÜ)ˆLÜ˜×*Ñ*Ó,Ó-ˆDØ˜A‘w—}‘} SÓ)¨!Ñ,¨sÑ2ˆHó �Ø ŸY™Y s›^�
Ø"(¨{°
Ð+;Ñ";�Ü˜d×0Ñ0Ó2Ó3°OÑD�Ü˜f¤n×5Ñ5Ø$¨;¨JÐ'7Ñ7¸-ÑGÈ#ÈkÐJZÑZ‘Gà$™l�GØ4;˜cÑ" =Ñ1Ø)6Ñ);�Ñ&Ø!Ò&ñ ô ›ˆJä × 2Ñ 2Ó 4×5�Ø%Ñ.Ð/AÑB�ô "& e§k¡k´"·&±&Ó&9Ó!:�Ü(¨Ö8‘G�A�tØ˜t•|Ø& qÓ)¨WÑ4Ó)Ùñ  9ô ŸG™GŸLšL¨.Ð9�	àÓ&ÙØ�	Ü˜u°Ò5¸Ø$%§F¡F¦H˜Ø&'§l¡l°8Ó&<˜Ø#$§:¡:£<˜Ø%-§^¡^°CÓ%8˜
Ø*0Ñ*;¸hÀ{È
Ð>SÑ*S˜Ü&*¨4×+=Ñ+=Ó+?Ó&@ÀÑ&Q˜Ü% k´>×BÑBØ(7¸-Ñ(GÈ(ÐS]ÐS^ÐJ_Ñ(_˜IØBR×BXÑBXÔBZÔ(uÒBZ±h°c¸3Ð^mÑ^t«¨¨cªÑBZ˜IÑ(uð 7@·o±oÖ6GÑ 2 ¨(Ø)1¯©¸Ó)< Ø*2°7°FÐ*;ÀÀ,ÀÐ>PÑ*PÐS[Ð\bÐ\cÐSdÑ*d Ø5= 	¨'Ó 2ò 7Hð
 )/Ñ(9¸HÑ(D˜IØ/:˜	 )Ó,ñ# %-ð$ —N‘N 9Ô-ð  )Ÿ~™~Ö/˜ØBKÐ\_Ñ-`˜Ó*ñ  0ô !#§¡§¡°	Ó :�IÜŸ7™7Ÿ>™>¨)×4Ñ4ÜŸš IÔ.Ü" 9¨iÀ(ÒK÷9 6Ò5ô 6ò' *ùóT )v÷ 6×5ús&   ÆBMÈ,L>È<L>ÉC(MÌ>MÍ
Mc                óØ  • UR                   (       a  U(       a  [        S5      eU/[        U R                  R	                  5       5      -   n[        U5      S:”  a‘  [        S U 5       5      (       a  Sn[        R                  " U5        g	[        S U 5       5      (       a  Sn[        R                  " U5        g	[        S U 5       5      (       a  Sn[        R                  " U5        g	g	g	)
z?Perform checks on newly added PEFT configs to ensure integrity.r   r/   c              3  óB   #   • U  H  n[        US S5      S:H  v •  M     g7f)r‡   NrŒ   r4  ©r€   rR   s     rh   r‚   Ú6PeftModel._check_new_adapter_config.<locals>.<genexpr>  s!   é € ÐcÒWbÈV”7˜6Ð#6¸Ó=ÀÖHÒWbùó   ‚züPiSSA changes the base weights of the model and should thus not be used with other adapters. Consider converting the PiSSA adapter into a normal LoRA adapter: https://github.com/huggingface/peft/tree/main/examples/pissa_finetuning#convert-pissa-to-lorac              3  óB   #   • U  H  n[        US S5      S:H  v •  M     g7f)r‡   Nr�   r4  r  s     rh   r‚   r    ó!   é € ÐeÒYdÈv”W˜VÐ%8¸$Ó?À7ÖJÒYdùr  züCorDA changes the base weights of the model and should thus not be used with other adapters. Consider converting the CorDA adapter into a normal LoRA adapter: https://github.com/huggingface/peft/tree/main/examples/corda_finetuning#convert-corda-to-lorac              3  óB   #   • U  H  n[        US S5      S:H  v •  M     g7f)r‡   NrŽ   r4  r  s     rh   r‚   r  $  r  r  zõOLoRA changes the base weights of the model and should thus not be used with other adapters. Consider converting the OLoRA adapter into a normal LoRA adapter: https://github.com/huggingface/peft/tree/main/examples/olora_finetuning#olora-and-loraN)	rY   r•   rs   rd   r  rÅ   r–   r—   r˜   )rb   rd   r  Úall_configsr£   s        rh   Ú_check_new_adapter_configÚ#PeftModel._check_new_adapter_config  sÔ   € à×)×)®lÜÐqÓrÐrð #�m¤d¨4×+;Ñ+;×+BÑ+BÓ+DÓ&EÑEˆÜˆ{Ó˜aÓÜÑcÑWbÓc×cÑcðtð ô
 —’˜cÕ"ÜÑeÑYdÓe×eÑeðtð ô
 —’˜cÕ"ÜÑeÑYdÓe×eÑeðmð ô
 —’˜cÕ"ð fð  ro   c	                ó(  • SSK Jn
  U R                  U	5      u  p¹Uc
  [        5       nX R                  ;  aZ  U
[
        R                  " U40 UD6   R                  " U4SU0UD6nU R                  XÃS9  U(       + Ul	        U R                  UUUUS9  [        U4XHS.UD6nU	R                  SS	5      n[        U UUUUS
9nU R                  U   R                  n[        R                  " US5      n/ nUR                    H,  nUU;   d  M  UU;   d  M  SU;   a  M  UR#                  U5        M.     UR                   R%                  5         UR                   R'                  U5        [)        U SS5      GbØ  [+        [-        U R.                  R1                  5       5      R3                  SS15      5      S:”  Ga–  [+        U R                  5      S:X  Ga|  U	R                  SS5      nU	R                  SS5      nU	R                  SS5      nU	R                  SS5      nU	R                  SS5      nUb  Ub  [5        S5      eUc  Un0 nS[6        R8                  " [:        5      R<                  ;   a  UUS'   U R>                  n[A        U[,        5      (       a  [C        U5      nUS:w  a  [E        U UUUS:H  S9n[A        U[F        5      (       a  [I        U UUS9nU RK                  UU5        UUS'   [;        U 4UUS.UD6  [M        SS9nU R                  U   RN                  (       a  [Q        U RR                  5        [U        U RW                  5       U5        [Y        U RZ                  S5      (       a  U RZ                  R]                  X%S 9  U(       d  U R_                  5         U$ )!a 	  
Load a trained adapter into the model.

The name for the new adapter should be unique.

The new adapter is not automatically set as the active adapter. Use [`PeftModel.set_adapter`] to set the active
adapter.

Args:
    model_id (`str` or `os.PathLike`):
        The name of the PEFT configuration to use. Can be either:
            - A string, the `model id` of a PEFT configuration hosted inside a model repo on the Hugging Face
              Hub.
            - A path to a directory containing a PEFT configuration file saved using the `save_pretrained`
              method (`./my_peft_config_directory/`).
    adapter_name (`str`):
        The name of the adapter to be added.
    is_trainable (`bool`, *optional*, defaults to `False`):
        Whether the adapter should be trainable or not. If `False`, the adapter will be frozen and can only be
        used for inference.
    torch_device (`str`, *optional*, defaults to None):
        The device to load the adapter on. If `None`, the device will be inferred.
    autocast_adapter_dtype (`bool`, *optional*, defaults to `True`):
        Whether to autocast the adapter dtype. Defaults to `True`. Right now, this will only cast adapter
        weights using float16 and bfloat16 to float32, as this is typically required for stable training, and
        only affect select PEFT tuners. If set to `False`, the dtypes will stay the same as those of the
        corresponding layer.
    ephemeral_gpu_offload (`bool`, *optional*, defaults to `False`):
        Whether to use ephemeral GPU offloading for partially loaded modules. Defaults to `False`.
    low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
        Create empty adapter weights on meta device before loading the saved weights. Useful to speed up the
        process.
    key_mapping (dict, *optional*, defaults to None)
        Extra mapping of PEFT `state_dict` keys applied before loading the `state_dict`. When this mapping is
        applied, the PEFT-specific `"base_model.model"` prefix is removed beforehand and the adapter name (e.g.
        `"default"`) is not inserted yet. Only pass this argument if you know what you're doing.
    kwargs: (`optional`):
        Additional arguments to modify the way the adapter is loaded, e.g. the token for Hugging Face Hub.
r/   )r2   Nr  )r  )rL   rP   )r  r  Úignore_mismatched_sizesF)rO   r  rL   rú   r  rò   rþ   rÿ   r   Ú
device_mapr  Ú
max_memoryÚoffload_folderÚoffload_dirrý   z<Cannot use `offload_folder` when `offload_dir` is specified.Ú
sequentialÚbalanced_low_0)r  Úno_split_module_classesÚlow_zero)r  r  )r  r  T)Úio_same_devicerM   rN   )0Úmappingr2   rí  r@   rd   r1   r  r  r  rÏ   r]   rA   r¼   rC   rW   r3   r!  rÃ   ÚclearÚextendr_   rÅ   r  rò   r  r  r•   rè  ré  r   r:  Ú_no_split_modulesrr   rs   r   rt   r   r
  r   rY   r   r  r   rJ  r^   r\   rM   Úeval)rb   r"  rO   r  Útorch_devicerP   r  rL   r  r¢   r2   rë  rd   rõ  r  r+  ÚtunerÚtuner_prefixÚadapter_missing_keysrå   r  r  r  r  rý   Údispatch_model_kwargsr  Úhooks                               rh   rœ   ÚPeftModel.load_adapter,  s½  € õf 	9à)-×);Ñ);¸FÓ)CÑ&ÐØÑÜ'›>ˆLà×/Ñ/Ó/à5Ü×)Ò)Øñà,ññ÷
 ‰oðð ñ	ð '<ð	ð )ñ	ˆKð ×*Ñ*¨;Ð*ÑRØ-9Ô)9ˆKÔ&Ø×ÑØØØ"3Ø'=ð	 ñ ô -Øð
Ø)ñ
ØF\ñ
Ðð
 #)§*¡*Ð-FÈÓ"NÐÜ/ØØØ%Ø$;Ø/ñ
ˆð × Ñ  Ñ.×8Ñ8ˆÜ2×6Ò6°u¸bÓAˆØ!Ðð ×+Ô+ˆCØ˜sÕ" |°sÕ':à! SÓ(ÙØ$×+Ñ+¨CÖ0ñ ,ð 	× Ñ ×&Ñ&Ô(Ø× Ñ ×'Ñ'Ð(<Ô=ô �T˜?¨DÓ1Ò=Ü”S˜×+Ñ+×2Ñ2Ó4Ó5×BÑBÀEÈ6À?ÓSÓTÐWXÔXÜ�D×$Ñ$Ó%¨Ô*àŸ™ L°&Ó9ˆJØŸ™ L°$Ó7ˆJØ#ŸZ™ZÐ(8¸$Ó?ˆNØ Ÿ*™* ]°DÓ9ˆKØ"ŸJ™J ¸Ó=ˆMàÑ&¨>Ñ+Eä Ð!_Ó`Ð`ØÑ$à,�à$&Ð!ð ¤'×"3Ò"3´NÓ"C×"NÑ"NÓNØ9FÐ% oÑ6à&*×&<Ñ&<Ð#ÜÐ1´3×7Ñ7Ü*.Ð/FÓ*GÐ'à˜\Ó)Ü0ØØ)Ø,CØ(Ð,<Ñ<ñ	�
ô ˜*¤c×*Ñ*Ü2Ø ZÐI`ñ�
ð × Ñ  Ð0@ÔAØ5BÐ! /Ñ2äØðà%Ø'ñð (ò	ô $°4Ñ8ˆDØ×Ñ Ñ-×@×@Ü+¨D×,?Ñ,?Ô@Ü˜t×2Ñ2Ó4°dÔ;ä�4—?‘?Ð$9×:Ñ:Ø�O‰O×/Ñ/Ø)ð 0ñ ö
 Ø�I‰IŒKØÐro   c                óÐ   • XR                   ;  a  [        SU S35      eXl        U R                   U   R                  (       d  U R                  R                  XS9  g[        XUS9  g)aÎ  
Sets the active adapter.

Only one adapter can be active at a time.

Additionally, this function will set the specified adapter to trainable (i.e., requires_grad=True) unless
inference_mode is True.

Args:
    adapter_name (`str`):
        The name of the adapter to be set as active. The adapter must be loaded first.
    inference_mode (`bool`, optional):
        Whether the activated adapter should be frozen (i.e. `requires_grad=False`). Default is False.
rÛ  z not found.)rÏ   N)rd   r•   rV   rY   r\   Úset_adapterr<   )rb   rO   rÏ   s      rh   r/  ÚPeftModel.set_adapterÙ  s_   € ð ×/Ñ/Ó/Ü˜x¨ ~°[ÐAÓBÐBØ*ÔØ×Ñ Ñ-×@×@à�O‰O×'Ñ'¨Ð'ÒTô ˜¸NÓKro   c                óÆ   • U R                   R                  (       a-  [        SU R                   R                  R                   S35      eU R
                  R                  XS9  g)ag  
Enable or disable gradients on the given adapter(s).

Note: Not supported for prompt learning methods like prompt tuning.

Args:
    adapter_name (`str` or `Sequence[str]`):
        The name of the adapter(s) whose gradients should be enabled/disabled.
    requires_grad (`bool`, *optional*)
        Whether to enable (`True`, default) or disable (`False`).
zJSetting `requires_grad` is not supported for prompt learning methods like r„   )rI   r;  N)r„  rY   r  rW   r|   r\   Úset_requires_grad)rb   rI   r;  s      rh   r2  ÚPeftModel.set_requires_gradò  sY   € ð ×"Ñ"×5×5ÜØ\Ø×*Ñ*×4Ñ4×:Ñ:Ð;¸1ð>óð ð
 	�‰×)Ñ)¸Ð)Òcro   c                ó0   • [        U R                  SS 5      $ )Nrü   )r_   r\   rm   s    rh   r†  Ú PeftModel.base_model_torch_dtype  s   € ä�t—‘¨°Ó6Ð6ro   c                ó4   • U R                   U R                     $ rk   )rd   rV   rm   s    rh   r„  ÚPeftModel.active_peft_config
  s   € à×Ñ × 3Ñ 3Ñ4Ð4ro   c                óê  • U R                   R                  n[        U[        5      (       d  UR                  n/ n[        U R                  S5      (       aD  [        U R                  R                  [        R                  5      (       a  UR                  S5        US:X  a  UR                  S5        [        U R                  S5      (       a(  UR                  SU R                  R                   35        U$ )zÓDerive tags for the model card from the adapter's config. For example, setting the
base model is important for enabling support for HF inference providers but it also makes models more
searchable on the HF hub.
rc   r‘  ÚLORAÚlorar³   zbase_model:adapter:)r„  rW   rr   rt   r|   r^   r\   rc   r‘  r    rÃ   r³   )rb   Úpeft_methodÚtagss      rh   Ú_get_peft_specific_model_tagsÚ'PeftModel._get_peft_specific_model_tags  sµ   € ð
 ×-Ñ-×7Ñ7ˆÜ˜+¤s×+Ñ+Ø%×+Ñ+ˆKàˆä�4—?‘? G×,Ñ,´¸D¿O¹O×<QÑ<QÔS_×SoÑSo×1pÑ1pØ�K‰K˜Ô'à˜&Ó Ø�K‰K˜Ôä�4—?‘? N×3Ñ3Ø�K‰KÐ-¨d¯o©o×.JÑ.JÐ-KÐLÔMàˆro   c                ó.  • [         R                  R                  US5      n[         R                  R                  U5      (       a  [        R
                  " U5      O[        R                  " [        5       5      nSUR                  S'   [        5       nU R                  5       n[        US5      (       a$  UR                  UR                  =(       d    / 5      nUR                  U R                  5       5      nU(       a  [        U5      UR                  S'   UR                  R                   (       d&  [#        U [$        5      (       a  SUR                  l        [&        R(                  " U 5      nU[*        :X  a  SOUnUb  SU;   a  US   UR                  S	'   UR,                  R/                  5       nSn[        US
5      (       a$  U R0                  R2                  R5                  5       nSn	Sn
UbK  U	SU
 S3-  n	U	SR                  UR7                  5        VVs/ s H  u  p¼SU SU 3PM     snn5      -  n	U	S-  n	SnX§;  aP  [9        U	5      (       a@  X×;   a%  UR;                  UR=                  U5      S-   U	5        OUR?                  U SU	 35        SnS[@         3U;  aK  Xç;   a,  UR;                  UR=                  U5      S-   S[@         35        OUR?                  U S[@         35        SR                  U5      Ul        URC                  U5        gs  snnf )z¹
Updates or create model card to include information about peft:
1. Adds `peft` library tag
2. Adds peft version
3. Adds base model info
4. Adds quantization information if it was used
z	README.mdÚpeftÚlibrary_nameÚ
model_tagsr<  ztext-generationNÚ_name_or_pathr\   Úquantization_configrú   zJThe following `bitsandbytes` quantization config was used during training:Ú
z- z: z## Training procedurer·   z### Framework versionsz- PEFT z	

- PEFT )"r™   rš   r½   r  r   ÚloadÚfrom_templater   Údatar  rJ  r^   ÚunionrB  r=  ÚsortedÚpipeline_tagrr   ÚPeftModelForCausalLMr'   rÖ  r*   ÚtextÚ
splitlinesrR   rD  Úto_dictrÀ   ÚboolÚinsertr  rÃ   r0   rÌ   )rb   rÚ   ÚfilenameÚcardr<  r\   r™  ÚlinesrD  Útraining_config_textÚquantization_prefixrÜ   r|   Útraining_procedure_headingÚframework_block_headings                  rh   r»   Ú%PeftModel.create_or_update_model_card$  s©  € ô —7‘7—<‘< 
¨KÓ8ˆä+-¯7©7¯>©>¸(×+CÑ+CŒy�~Š~˜hÔ'Ì×I`ÒI`ÔanÓapÓIqˆà$*ˆ�	‰	�.Ñ!ä‹uˆØ×(Ñ(Ó*ˆ
Ü�:˜|×,Ñ,Ø—:‘:˜j×3Ñ3×9°rÓ:ˆDà�z‰z˜$×<Ñ<Ó>Ó?ˆÞÜ & t£ˆD�I‰I�fÑð �y‰y×%×%¬*°TÔ;O×*PÑ*PØ%6ˆD�I‰IÔ"ä ×1Ò1°$Ó7ˆØ+Ô/AÓA‘tÀ|ˆØÑ#¨¸<Ó(GØ&2°?Ñ&CˆD�I‰I�lÑ#à—	‘	×$Ñ$Ó&ˆà"ÐÜ�<Ð!6×7Ñ7Ø"&§+¡+×"AÑ"A×"IÑ"IÓ"KÐØ!ÐØjÐàÑ*Ø  bÐ)<Ð(=¸RÐ$@Ñ@Ð Ø  D§I¡IÐWj×WpÑWpÔWrÔ.sÒWrÉÈ°°D°6¸¸E¸7Ó/CÑWrÒ.sÓ$tÑtÐ Ø  DÑ(Ð à%<Ð"ØÓ+´Ð5I×0JÑ0JØ)Ó2Ø—‘˜UŸ[™[Ð)CÓDÀqÑHÐJ^Õ_à—‘Ð :Ð;¸2Ð>RÐ=SÐTÔUð #;ÐØ”[�MÐ"¨%Ó/Ø&Ó/Ø—‘˜UŸ[™[Ð)@ÓAÀAÑEÈÔQ\ÐP]ÐG^Õ_à—‘Ð 7Ð8¸ÄKÀ=ÐQÔRà—I‘I˜eÓ$ˆŒ	Ø�	‰	�(Õùó' /ts   ÈLc                óª   • U R                   nUR                  (       a  g[        U R                  S5      (       d  gU R                  R	                  5       $ )z¦
Whether it is possible for the adapter of this model to be converted to LoRA.

Normally, this works if the PEFT method is additive, i.e. W' = W_base + delta_weight.
FÚsupports_lora_conversion)r„  rY   r^   r\   r[  )rb   rO   rd   s      rh   r[  Ú"PeftModel.supports_lora_conversione  sC   € ð ×-Ñ-ˆØ×)×)Øä�t—‘Ð(B×CÑCØà�‰×7Ñ7Ó9Ð9ro   )ra   rZ   r[   rV   r\   ro  rW   rÇ  r  r8  rX   r<  rC  )r®   TF)rc   r    rd   r1   rO   rt   rP   rP  rL   rP  ÚreturnÚNone)r]  údict[str, PeftConfig])r]  ú	list[str])r]  rP  )r|   r_  )TNr  TN)rÖ   rt   r×   rP  rØ   zOptional[list[str]]r­   zUnion[str, bool]rÙ   rP  r    úOptional[str]r¢   r	   r]  r^  )r®   FNTFFN)rc   útorch.nn.Moduler"  úUnion[str, os.PathLike]rO   rt   r  rP  rR   zOptional[PeftConfig]rP   rP  r  rP  rL   rP  r  úOptional[dict[str, str]]r¢   r	   r]  rF   )rO   rt   )rc   r    )rO   rt   r]  útorch.Tensor)NN)r–  Úintr—  úOptional[torch.Tensor]r  zOptional[int]r]  re  )r]  ztuple[int, int])r]  r^  )rÜ   rt   )r»  r	   r¢   r	   )F)r]  rb  ©FT©
rO   rt   rd   r1   rL   rP  rP   rP  r]  r^  )rO   rt   r]  r^  )r]  zOptional[set[str]])r]  úlist[TunerLayerStatus])r]  ÚTunerModelStatus)r¢   zdict[str, Any])rý   zdict[str, dict[str, str]]rõ  zdict[str, torch.tensor])rd   r1   r  rP  r]  r^  )FNTFFN)r"  rc  rO   rt   r  rP  r'  ra  rP   rP  r  rP  rL   rP  r  rd  r¢   r	   )rO   rt   rÏ   rP  r]  r^  )T)rI   zstr | Sequence[str]r;  rP  r]  r^  )rÚ   rt   ©r®   )rO   rt   r]  rP  )/rÓ   rÒ   Ú__qualname__Ú__firstlineno__Ú__doc__rU   Úpropertyrd   rq   ry   ÚsetterrÈ   Úclassmethodr  rV  r`   rY  rs  r¢  r¯  r´  r·  r   rº  ro  rÁ  rÑ   rÐ  rJ  r]   rŸ   rà  rã  rË  rí  r
  r  rœ   r/  r2  r†  r„  r=  r»   r[  Ú__static_attributes__Ú__classcell__©rg   s   @rh   rF   rF   H   sª  ø† ñðF &Ø'+Ø"'ð)(àð)(ð  ð)(ð ð	)(ð
 !%ð)(ð  ð)(ð 
÷)(ð )(ðV ó+ó ð+ð
 óó ðð$ óGó ðGð ×Ñó0ó ð0ð $(Ø15Ø28Ø $ØBFðN8àðN8ð !ðN8ð /ð	N8ð
  0ðN8ð ðN8ð 3@ðN8ð ðN8ð 
õN8ð` ð
 &Ø"Ø'+Ø'+Ø&+Ø"'Ø04ðaàðað *ðað ð	að
 ðað %ðað !%ðað  $ðað  ðað .ðað ðað 
ôaó ðaôFJôX>ô÷ 3ð, hlð{Øð{Ø)?ð{ØWdð{à	õ{ôz+ô<
÷"2ð ñ
ó ð
ô:òCô
)ð ñ30ó ð30ôjhð #(Ø'+ðLàðLð  ðLð  ð	Lð
 !%ðLð 
õLô\9ð& ó
ó ð
ô&ô<&&ðP ó4ó ð4ôNLô`#ðD #Ø&*Ø'+Ø&+Ø"'Ø04ðkà)ðkð ðkð ð	kð
 $ðkð !%ðkð  $ðkð  ðkð .ðkð õköZLö2dð( ñ7ó ð7ð ñ5ó ð5òô,?÷B:ô :ro   rF   c                  ó    ^ • \ rS rSrSr S       S	U 4S jjjr  S
         SU 4S jjjr        SS jr       SS jrSr	U =r
$ )Ú"PeftModelForSequenceClassificationiu  a7  
Peft model for sequence classification tasks.

Args:
    model ([`~transformers.PreTrainedModel`]): Base transformer model.
    peft_config ([`PeftConfig`]): Peft config.
    adapter_name (`str`,  *optional*): The name of the adapter, defaults to `"default"`.
    autocast_adapter_dtype (`bool`, *optional*, defaults to `True`):
        Whether to autocast the adapter dtype. Defaults to `True`. Right now, this will only cast adapter weights
        using float16 and bfloat16 to float32, as this is typically required for stable training, and only affect
        select PEFT tuners. If set to `False`, the dtypes will stay the same as those of the corresponding layer.

**Attributes**:
    - **config** ([`~transformers.PretrainedConfig`]) -- The configuration object of the base model.
    - **cls_layer_name** (`str`) -- The name of the classification layer.

Example:

    ```py
    >>> from transformers import AutoModelForSequenceClassification
    >>> from peft import PeftModelForSequenceClassification, get_peft_config

    >>> config = {
    ...     "peft_type": "PREFIX_TUNING",
    ...     "task_type": "SEQ_CLS",
    ...     "inference_mode": False,
    ...     "num_virtual_tokens": 20,
    ...     "token_dim": 768,
    ...     "num_transformer_submodules": 1,
    ...     "num_attention_heads": 12,
    ...     "num_layers": 12,
    ...     "encoder_hidden_size": 768,
    ...     "prefix_projection": False,
    ...     "postprocess_past_key_value_function": None,
    ... }

    >>> peft_config = get_peft_config(config)
    >>> model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased")
    >>> peft_model = PeftModelForSequenceClassification(model, peft_config)
    >>> peft_model.print_trainable_parameters()
    trainable params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117
    ```
c           	     óÂ  >^• SS/n[        US5      (       a3  UR                  c  US S  Ul        OUR                  R                  U5        [        TU ]  " XU40 UD6  [        US5      (       aQ  U R
                  R                  5        H3  u  mn[        U4S jU R                   5       5      (       d  M,  TU l          O   [        U U[        USS 5      UR                  S9  g )NÚ
classifierÚscorerà  c              3  ó,   >#   • U  H	  oT;   v •  M     g 7frk   r	  ©r€   Úmodule_namerÜ   s     €rh   r‚   Ú>PeftModelForSequenceClassification.__init__.<locals>.<genexpr>´  s   øé € ÐSÒ>R¨{ dÖ*Ò>Rùó   ƒ©Úmodule_namesrÏ   )r^   rà  r$  rT   rU   r\   r9  r–   Úcls_layer_namer=   r_   rÏ   ©	rb   rc   rd   rO   r¢   Úclassifier_module_namesrá   rÜ   rg   s	          @€rh   rU   Ú+PeftModelForSequenceClassification.__init__¢  sÓ   ù€ ð $0°Ð"9Ðä�;Ð 1×2Ñ2Ø×*Ñ*Ñ2Ø.EÁaÐ.H�Õ+à×+Ñ+×2Ñ2Ð3JÔKô
 	‰Ò˜¨\ÑD¸VÒDä�;Ð 1×2Ñ2ØŸ?™?×9Ñ9Ö;‘��aÜÔS¸d×>RÒ>RÓS×SÓSØ*.�DÔ'Ùñ <ô 	ØØÜ  Ð.?ÀÓFØ&×5Ñ5ó		
ro   c                ó°   >• [        US5      (       a7  SS/nUR                  c  USS Ul        OUR                  R                  U5        [        TU ]  XUS9$ )rÕ  rà  ry  rz  NrK   ©r^   rà  r$  rT   r]   ©rb   rO   rd   rL   rP   r„  rg   s         €rh   r]   Ú.PeftModelForSequenceClassification.add_adapterÀ  sc   ø€ ô@ �;Ð 1×2Ñ2Ø'3°WÐ&=Ð#Ø×*Ñ*Ñ2Ø.EÁaÐ.H�Õ+à×+Ñ+×2Ñ2Ð3JÔKä‰wÑ" <ÐPaÐ"ÐbÐbro   c	                óî  • Ub  UOU R                   R                  nU R                  n
U
R                  (       d�  U R                  " S0 U	D6   U	R                  5        VVs0 s H  u  p¼X°R                  ;  d  M  X¼_M     n	nnU
R                  [        R                  :X  a  X‰S'   U R                  " SUUUUUUUS.U	D6sS S S 5        $ [        X5      nUbO  [        R                  " XÚR                  5      R                  UR                   5      n[        R"                  " Xâ4SS9nU	R%                  SS 5      b_  U
R                  [        R&                  [        R(                  4;   a  U	S   U
R                  -   U	S'   O[*        R,                  " S5        S U	S'   U	R/                  UUUUUS.5        U
R                  [        R&                  [        R(                  4;   a  U R0                  " SSU0U	D6$ U	R%                  S	S 5      bv  [        R"                  " [        R2                  " XÚR                  5      R                  U R4                  R6                  R                   5      U	S	   4SS9R9                  5       U	S	'   Uc  U R5                  U5      nU R;                  XØS
9nUR                  UR<                  5      n[        R"                  " Xó4SS9nU R                  " SSU0U	D6$ s  snnf ! , (       d  f       GN.= f©Nr—  ©Ú	input_idsÚattention_maskÚinputs_embedsÚlabelsÚoutput_attentionsÚoutput_hidden_statesÚreturn_dictr/   rv  Úposition_idsúUPosition ids are not supported for parameter efficient tuning. Ignoring position ids.©rŽ  r�  r‘  r’  r“  r�  Útoken_type_ids©r–  r—  r�  r	  )rR   Úuse_return_dictr„  rY   rº  rÀ   rX   rW   r8   ÚPOLYr\   r:   rÁ   ÚonesrO  rL  r  r‹  r¼   rH  rI  r—   r˜   rM  Ú_prefix_tuning_forwardÚzerosrC  rn  rP  r¢  rü   ©rb   r�  rŽ  r�  r�  r‘  r’  r“  r—  r¢   rd   rã   rä   r–  Úprefix_attention_maskr¡  s                   rh   ro  Ú*PeftModelForSequenceClassification.forwardé  s°  € ð &1Ñ%<‘kÀ$Ç+Á+×B]ÑB]ˆØ×-Ñ-ˆØ×-×-Ø×0Ò0Ñ:°6Ó:Ø+1¯<©<¬>Ôeª>¡4 1¸Q×FdÑFdÑ=d›$˜!š$©>�ÑeØ×(Ñ(¬H¯M©MÓ9Ø)1˜:Ñ&Ø—’ð 	Ø'Ø#1Ø"/Ø!Ø&7Ø)=Ø +ñ	ð ñ	÷	 ;Ñ:ô % YÓ>ˆ
ØÑ%ä$)§J¢J¨z×;YÑ;YÓ$Z×$]Ñ$]Ð^l×^sÑ^sÓ$tÐ!Ü"ŸYšYÐ(=Ð'NÐTUÑVˆNØ�:‰:�n dÓ+Ñ7Ø×$Ñ$¬×)?Ñ)?Ä×ASÑASÐ(TÓTà)/°Ñ)?À+×B`ÑB`Ñ)`��~Ò&ä—’ÐuÔvØ)-��~Ñ&Ø�‰à"0Ø Ø%6Ø(<Ø*ñô	
ð × Ñ ¤X×%;Ñ%;¼X×=OÑ=OÐ$PÓPØ×.Ò.ÑM¸ÐMÀfÑMÐMà�z‰zÐ*¨DÓ1Ñ=Ü+0¯9ª9äŸš J×0NÑ0NÓO×RÑRÐSW×SgÑSg×SnÑSn×SuÑSuÓvØÐ/Ñ0ðð ñ,÷ ‘$“&ð Ð'Ñ(ð Ñ$Ø $× 4Ñ 4°YÓ ?�Ø—o‘o°�oÐOˆGØ—j‘j ×!4Ñ!4Ó5ˆGÜ!ŸIšI wÐ&>ÀAÑFˆMØ—?’?ÑI°ÐIÀ&ÑIÐIùói f÷ ;Ö:úó$   ÁK%ÁKÁ8KÁ>>K%ËK%Ë%
K4c           
     ój  • [        X5      n	U R                  U	5      n
[        [        R                  " U R
                  R                  5      R                  R                  5       5      nUR                  UUUUUUU
S.5        SU;   a  U R
                  " SSU0UD6$ U R
                  R                  U R                  5      n[        [        R                  " UR                  5      R                  R                  5       5      nSU;  a  [        S5      eU" S0 UD6n[        U5      S:”  a  US   OUS   nS[        U R
                  R                  5       5       VVs/ s H  u  nnUPM
     snn;   a  U R
                  R                  U5      nU R
                  R                  U R                   5      " U5      nS nUGb¸  U R"                  R$                  c¥  U R
                  R&                  S:X  a  SU R"                  l        OyU R
                  R&                  S:”  aN  UR(                  [*        R,                  :X  d  UR(                  [*        R.                  :X  a  S	U R"                  l        OS
U R"                  l        U R"                  R$                  S:X  aT  [1        5       nU R
                  R&                  S:X  a&  U" UR3                  5       UR3                  5       5      nO˜U" UU5      nOŽU R"                  R$                  S	:X  aG  [5        5       nU" UR7                  SU R
                  R&                  5      UR7                  S5      5      nO-U R"                  R$                  S
:X  a  [9        5       nU" UU5      nU(       d  U4USS  -   nUb  U4U-   $ U$ [;        UUUR<                  UR>                  S9$ s  snnf )N©r�  rŽ  r�  r‘  r’  r“  r˜  r˜  r�  úLModel does not support past key values which are required for prefix tuning.r/   r   ÚdropoutÚ
regressionÚsingle_label_classificationÚmulti_label_classificationrj  r·   ©ÚlossÚlogitsÚhidden_statesÚ
attentionsr	  ) r:   r¢  rs   rè  ré  r\   ro  r:  r   rM  r?  r<  r•   rÅ   r9  r¥  r‚  rR   Úproblem_typeÚ
num_labelsrü   rÁ   rP  rf  r   Úsqueezer   r‡  r   r"   r¬  r­  )rb   r�  rŽ  r�  r�  r‘  r’  r“  r¢   r–  r˜  Ú
fwd_paramsr<  ÚoutputsÚpooled_outputrÜ   rá   r«  rª  Úloss_fctrd  s                        rh   rœ  Ú9PeftModelForSequenceClassification._prefix_tuning_forward/  s  € ô % YÓ>ˆ
ØŸ/™/¨*Ó5ˆÜœ'×+Ò+¨D¯O©O×,CÑ,CÓD×OÑO×TÑTÓVÓWˆ
Ø�‰à&Ø"0Ø!.Ø%6Ø(<Ø*Ø#2ñô
	
ð  
Ó*Ø—?’?Ñ;¨&Ð;°FÑ;Ð;à(,¯©×(EÑ(EÀd×FdÑFdÓ(eÐ%Üœg×/Ò/Ð0I×0QÑ0QÓR×]Ñ]×bÑbÓdÓeˆJØ ¨
Ó2Ü Ð!oÓpÐpÙ/Ñ9°&Ñ9ˆGÜ*-¨g«,¸Ó*:˜G AšJÀÈÁ
ˆMØ´°d·o±o×6TÑ6TÓ6VÔ1WÔXÒ1W¡g d¨A›TÑ1WÒXÓXØ $§¡× 7Ñ 7¸Ó F�Ø—_‘_×2Ñ2°4×3FÑ3FÔGÈÓVˆFàˆDØÒ!Ø—;‘;×+Ñ+Ñ3Ø—‘×1Ñ1°QÓ6Ø3?˜Ÿ™Õ0ØŸ™×3Ñ3°aÓ7¸V¿\¹\ÌUÏZÉZÓ=WÐ[a×[gÑ[gÔkp×ktÑktÓ[tØ3P˜Ÿ™Õ0à3O˜Ÿ™Ô0à—;‘;×+Ñ+¨|Ó;Ü&›y�HØ—‘×1Ñ1°QÓ6Ù'¨¯©Ó(8¸&¿.¹.Ó:JÓK™á'¨°Ó7™Ø—[‘[×-Ñ-Ð1NÓNÜ/Ó1�HÙ# F§K¡K°°D·O±O×4NÑ4NÓ$OÐQW×Q\ÑQ\Ð]_ÓQ`Óa‘DØ—[‘[×-Ñ-Ð1MÓMÜ0Ó2�HÙ# F¨FÓ3�DÞØ ˜ W¨Q¨R [Ñ0�Ø-1Ñ-=˜˜ &Ñ(ÐIÀ6ÐIä+ØØØ%×3Ñ3Ø"×-Ñ-ñ	ð ùó= Ys   ÅN/©r‚  rl  ©rc   rb  rd   r1   rO   rt   r]  r^  rh  ri  ©NNNNNNNN©NNNNNNN©rÓ   rÒ   rm  rn  ro  rU   r]   ro  rœ  rs  rt  ru  s   @rh   rw  rw  u  sÊ   ø† ñ*ðZ T]ð
Ø$ð
Ø3=ð
ØMPð
à	÷
ð 
ðD #(Ø'+ð'càð'cð  ð'cð  ð	'cð
 !%ð'cð 
÷'cð 'cðV ØØØØØ!ØØôDJðP ØØØØØ!Ø÷Eò Ero   rw  c                  ó|   ^ • \ rS rSrSr S       SU 4S jjjr        SS jrS rS rSS.SS	 jjr	S
r
U =r$ )rL  iw  a;  
Peft model for causal language modeling.

Args:
    model ([`~transformers.PreTrainedModel`]): Base transformer model.
    peft_config ([`PeftConfig`]): Peft config.
    adapter_name (`str`,  *optional*): The name of the adapter, defaults to `"default"`.
    autocast_adapter_dtype (`bool`, *optional*, defaults to `True`):
        Whether to autocast the adapter dtype. Defaults to `True`. Right now, this will only cast adapter weights
        using float16 and bfloat16 to float32, as this is typically required for stable training, and only affect
        select PEFT tuners. If set to `False`, the dtypes will stay the same as those of the corresponding layer.

Example:

    ```py
    >>> from transformers import AutoModelForCausalLM
    >>> from peft import PeftModelForCausalLM, get_peft_config

    >>> config = {
    ...     "peft_type": "PREFIX_TUNING",
    ...     "task_type": "CAUSAL_LM",
    ...     "inference_mode": False,
    ...     "num_virtual_tokens": 20,
    ...     "token_dim": 1280,
    ...     "num_transformer_submodules": 1,
    ...     "num_attention_heads": 20,
    ...     "num_layers": 36,
    ...     "encoder_hidden_size": 1280,
    ...     "prefix_projection": False,
    ...     "postprocess_past_key_value_function": None,
    ... }

    >>> peft_config = get_peft_config(config)
    >>> model = AutoModelForCausalLM.from_pretrained("gpt2-large")
    >>> peft_model = PeftModelForCausalLM(model, peft_config)
    >>> peft_model.print_trainable_parameters()
    trainable params: 1843200 || all params: 775873280 || trainable%: 0.23756456724479544
    ```
c                ó`   >• [         TU ]  " XU40 UD6  U R                  R                  U l        g rk   )rT   rU   r\   rÇ  Ú(base_model_prepare_inputs_for_generation©rb   rc   rd   rO   r¢   rg   s        €rh   rU   ÚPeftModelForCausalLM.__init__   s+   ø€ ô 	‰Ò˜¨\ÑD¸VÒDØ8<¿¹×8eÑ8eˆÕ5ro   Nc	                óf  • U R                   n
U
R                  (       då  [        XU40 U	D6n	U R                  R                  R
                  S:X  a'  Ub  [        S5      eU R                  " SUUUUUUS.U	D6$ U
R                  [        R                  :X  a  X‰S'   U R                  " S0 U	D6   U	R                  5        VVs0 s H  u  p¼X°R                  ;  d  M  X¼_M     n	nnU R                  " SUUUUUUUS.U	D6sS S S 5        $ [        X5      nUbO  [        R                  " XÚR                   5      R#                  UR$                  5      n[        R&                  " Xâ4SS9nU	R)                  SS 5      b_  U
R                  [        R*                  [        R,                  4;   a  U	S   U
R                   -   U	S'   O[.        R0                  " S	5        S U	S'   U	R)                  S
S 5      b  [.        R0                  " S5        S U	S
'   U	R3                  UUUUUS.5        U
R                  [        R*                  [        R,                  4;   ab  Ub  UR4                  S   U
R                   -   nOUR4                  S   U
R                   -   nU R7                  XßS9U	S'   U R                  " SXS.U	D6$ U
R                  [        R8                  :X  a  U R:                  " XX¨U40 U	D6$ Uc  U R=                  U5      nUbU  [        R>                  " XÚR                   4S5      R#                  UR$                  5      n[        R&                  " UU4SS9U	S'   U R7                  XØS9nUR#                  UR@                  5      n[        R&                  " UU4SS9nU R                  " SSU0U	D6$ s  snnf ! , (       d  f       GN­= f)NÚmptz8forward in MPTForCausalLM does not support inputs_embeds)r�  rŽ  r�  r‘  r’  r“  r—  rŒ  r/   rv  r”  r•  r—  úXToken type ids are not supported for parameter efficient tuning. Ignoring token type idsr–  )r  r˜  )r�  r�  éœÿÿÿr�  r˜  r�  r	  )!r„  rY   r%   r\   rR   rz  ÚAssertionErrorrW   r8   rš  rº  rÀ   rX   r:   rÁ   r›  rO  rL  r  r‹  r¼   rH  rI  r—   r˜   rM  rB  r¢  rF  Ú_cpt_forwardrC  Úfullrü   )rb   r�  rŽ  r�  r�  r‘  r’  r“  r—  r¢   rd   rã   rä   r–  rŸ  r  Úprefix_labelsr¡  s                     rh   ro  ÚPeftModelForCausalLM.forward¦  sŠ  € ð ×-Ñ-ˆà×-×-ä2°4ÀMÑ\ÐU[Ñ\ˆFØ�‰×%Ñ%×0Ñ0°EÓ9Ø Ñ,Ü(Ð)cÓdÐdØ—’ð Ø'Ø#1Ø!Ø&7Ø)=Ø +ñð ñð ð ×$Ñ$¬¯©Ó5Ø%-�zÑ"à×0Ò0Ñ:°6Ó:Ø+1¯<©<¬>Ôeª>¡4 1¸Q×FdÑFdÑ=d›$˜!š$©>�ÑeØ—’ð 	Ø'Ø#1Ø"/Ø!Ø&7Ø)=Ø +ñ	ð ñ	÷ ;Ñ:ô % YÓ>ˆ
ØÑ%ä$)§J¢J¨z×;YÑ;YÓ$Z×$]Ñ$]Ð^l×^sÑ^sÓ$tÐ!Ü"ŸYšYÐ(=Ð'NÐTUÑVˆNà�:‰:�n dÓ+Ñ7Ø×$Ñ$¬×)?Ñ)?Ä×ASÑASÐ(TÓTà)/°Ñ)?À+×B`ÑB`Ñ)`��~Ò&ä—’ÐuÔvØ)-��~Ñ&Ø�:‰:Ð&¨Ó-Ñ9Ü�MŠMÐtÔuØ'+ˆFÐ#Ñ$Ø�‰à"0Ø Ø%6Ø(<Ø*ñô	
ð × Ñ ¤X×%;Ñ%;¼X×=OÑ=OÐ$PÓPð Ñ$Ø )§¡°Ñ 2°[×5SÑ5SÑ S‘à -× 3Ñ 3°AÑ 6¸×9WÑ9WÑ W�Ø(,¯©¸
¨Ð(`ˆFÐ$Ñ%Ø—?’?Ð^¨YÑ^ÐW]Ñ^Ð^Ø×"Ñ"¤h§l¡lÓ2Ø×$Ò$ Y¸{ÐV`ÑkÐdjÑkÐkàÑ$Ø $× 4Ñ 4°YÓ ?�àÑ!Ü %§
¢
¨J×8VÑ8VÐ+WÐY]Ó ^× aÑ aÐbh×boÑboÓ p�Ü#(§9¢9¨m¸VÐ-DÈ!Ñ#L��xÑ Ø—o‘o°�oÐOˆGØ—j‘j ×!4Ñ!4Ó5ˆGÜ!ŸIšI w°Ð&>ÀAÑFˆMØ—?’?ÑI°ÐIÀ&ÑIÐIùów f÷ ;Ö:ús$   Â*N!Â>NÃNÃN!ÎN!Î!
N0c                óD  • UR                  S5      nXU4 Vs/ s H  oˆc  M  UR                  PM     snS   n	SUR                  5       ;   a!  UR                  S5      R                  U	5      n
OKUc  UR                  S   nOUR                  S   n[
        R                  " X[45      R                  U	5      S-  n
UR                  nUR                  nUc  U R                  U5      nU R                  XTS9nUR                  UR                  5      n[
        R                  " Xâ4SS9nS nUGb   [
        R                  " U5      R                  5       R                  SS5      nUR!                  US5      R                  UR                  5      n[
        R                  " UU4SS9n[
        R                  " U5      R                  5       R                  SS5      nUR!                  US5      R                  UR                  5      nU
nUUS:„  ==   UR#                  5       -  ss'   [
        R                  " UU4SS9nUS:„  US-  S:H  -  nS	UU) '   XöS'   U R$                  " SS
U0UD6nUc  U$ [&        UR(                     nUR+                  UUWU R,                  S   5      nU$ s  snf )Nr�  r   Úinput_type_maskr/   ry  r˜  rv  rj  rÃ  r�  r®   r	  )Úpopr  r   rL  rB  rÁ   r›  Úcpt_token_idsÚcpt_tokens_type_maskrC  r¢  rü   r‹  rÂ   rP  r‡  r…  Úmaxr\   r4   rW   Úcalculate_lossrd   )rb   r�  r�  rd   r—  r–  r¢   r�  rý  r  rÊ  ÚN_tokensrÌ  rÍ  r¡  Ú
cpt_labelsrÇ  Úprefix_type_maskÚadjusted_input_type_maskÚcpt_type_maskÚ
labels_idxÚbase_model_outputÚcpt_embeddings                          rh   rÅ  Ú!PeftModelForCausalLM._cpt_forward  s�  € à—‘˜HÓ%ˆØ%.¸vÑ$FÓXÒ$F˜q“(�!—(”(Ñ$FÑXÐYZÑ[ˆà §¡£Ó-Ø$Ÿj™jÐ):Ó;×>Ñ>¸vÓF‰OàÑ Ø(×.Ñ.¨qÑ1‘à$Ÿ?™?¨1Ñ-�Ü#Ÿjšj¨*Ð)?Ó@×CÑCÀFÓKÈaÑOˆOà#×1Ñ1ˆØ*×?Ñ?Ðð Ñ Ø ×0Ñ0°Ó;ˆMà—/‘/¨Z�/ÐKˆØ—*‘*˜]×0Ñ0Ó1ˆÜŸ	š	 7Ð":ÀÑBˆàˆ
ØÒä!ŸLšL¨Ó7×<Ñ<Ó>×CÑCÀAÀrÓJˆMØ)×0Ñ0°¸QÓ?×BÑBÀ6Ç=Á=ÓQˆMÜŸš M°6Ð#:ÀÑBˆJä$Ÿ|š|Ð,@ÓA×FÑFÓH×MÑMÈaÐQSÓTÐØ/×6Ñ6°zÀ1ÓE×HÑHÈÏÉÓWÐØ'6Ð$Ø$Ð%=ÀÑ%AÓBÐFV×FZÑFZÓF\Ñ\ÓBä!ŸIšIÐ'7Ð9QÐ&RÐXYÑZˆMà'¨!Ñ+°ÀÑ0AÀQÑ0FÑGˆJØ&*ˆJ˜
�{Ñ#ð &ˆxÑà ŸOšOÑR¸-ÐRÈ6ÑRÐØ‰>Ø$Ð$ô 7°{×7LÑ7LÑMˆMØ -× <Ñ <Ø! :¨}¸d×>NÑ>NÈyÑ>Yó!Ðð %Ð$ùòg Ys
   ˜J¢Jc                ó  • U R                   nU R                  U R                  l        [        U R                  S5      (       a&  U R                  U R                  R
                  l        OU R                  U R                  l         UR                  (       d|  [        U /UQ70 UD6nU R                  " U0 UD6   UR                  5        VVs0 s H  u  pEX@R                  ;  d  M  XE_M     nnnU R                  R                  " U0 UD6nS S S 5        OU R                  R                  " U0 UD6nU R                  U R                  l        W$ s  snnf ! , (       d  f       N1= f!   U R                  U R                  l        e = f)Nrc   )r„  rÇ  r\   r^   Úgeneration_configrc   rY   r&   rº  rÀ   rX   rÁ  r½  )rb   r»  r¢   rd   rã   rä   r²  s          rh   rÁ  ÚPeftModelForCausalLM.generate=  s<  € Ø×-Ñ-ˆØ8<×8ZÑ8Zˆ�‰Ô5Ü�4—?‘? G×,Ñ,Ø6:×6LÑ6LˆD�O‰O×!Ñ!Õ3à04×0FÑ0FˆD�O‰OÔ-ð	Ø×1×1ä7¸ÐN¸tÒNÀvÑN�Ø×4Ò4°dÐE¸fÓEØ/5¯|©|¬~Ôiª~¡t qÀ×JhÑJhÑAh›d˜ašd©~�FÑiØ"Ÿo™o×6Ò6¸ÐGÀÑG�G÷ FÐEð Ÿ/™/×2Ò2°DÐC¸FÑC�ð
 =A×<iÑ<iˆD�O‰OÔ9ØˆNùó j÷ FÕEûð
	Ø<@×<iÑ<iˆD�O‰OÔ9Øús<   Â2E" Â7EÃEÃ$EÃ*EÄ	%E" ÅEÅ
EÅE" Å"F ©r—  c               óT  • U R                   nU R                  " U0 UD6n[        R                  R	                  [
        R                  5      [        R                  R	                  S5      :¬  n[        R                  R	                  [
        R                  5      [        R                  R	                  S5      :¬  n/ SQn[        R                  R	                  [
        R                  5      [        R                  R	                  S5      :”  a  UR                  S5        U=(       d,    U=(       a#    U R                  R                  R                  U;   n	UR                  S5      S L=(       a    US   S   S:H  n
UR                  [        R                  :X  a  XS'   UR                  (       Gap  U	(       a  UR                  S	S 5      bl  US	   n[!        U["        [$        45      (       a  US   S   R&                  S
   nOUR)                  5       nXÅS   R&                  S   :¼  a  US   S S 2SS 24   US'   UR                  SS 5      =nGbÄ  [!        U[*        5      (       aC  [-        U5      S:w  a  [/        S[-        U5       S35      e[%        UR1                  5       5      S   nUS   R&                  S   UR2                  4n[4        R6                  " U5      R9                  US   R:                  5      nUR=                  5       S:X  aï  UR&                  S   nUR&                  S   UR&                  S   -   n[4        R6                  " UU4UR>                  S9nU
(       aQ  UR                  [        R@                  [        RB                  4;  a#  [4        RD                  " UUS   R:                  S9nOUS   n[G        U RI                  5       S UUR                  S	5      UUUUR                  SS 5      S9nUUS'   O[4        RJ                  " Xý4SS9US'   UR                  SS 5      b_  UR                  [        R@                  [        RB                  4;   a  US   UR2                  -   US'   O[L        RN                  " S5        S US'   UR                  SS 5      b  [L        RN                  " S5        S US'   UR                  S	S 5      nUS L =(       d6    [!        U[
        RP                  5      =(       a    UR)                  5       (       + nU(       a¡  UR                  [        R@                  [        RB                  4;   as  [!        U[
        RP                  5      (       a,  [!        U[
        RR                  5      (       d  URT                  nOSnU RW                  US   R&                  S   US9nUUS	'   OuU(       an  U RY                  US   5      nU RW                  US   R&                  S   US9nUR9                  UR>                  5      n[4        RJ                  " UU4SS9US'   S US'   U
(       aG  UR                  [        R@                  [        RB                  4;   a  US==   UR2                  -  ss'   U$ UR                  [        R@                  [        RB                  4;  a  UR[                  SS 5      nU$ )Nz4.38.0z4.36.0)ÚllamaÚmistralÚ	persimmonÚphiz4.43.3Úbloomr�  r   r—  r˜  éþÿÿÿr�  r/   rj  rŽ  z&Expected a single attention mask, got za instead, please open an issue (https://github.com/huggingface/peft/issues) and report the error.ry  r·   )rü   r€  r”  )Úmodel_inputrŽ  r˜  r�  r–  Úsequence_lengthr”  rv  r•  r—  rÂ  )r–  r  r˜  r�  ).r„  r½  rŽ  r�  r�  r‘  r0   rÃ   r\   rR   rz  r¼   rW   r8   rš  rY   rr   Útuplers   rB  Úget_seq_lengthr  rÅ   r•   r  rO  rÁ   r›  rL  r  rw  rü   rH  rI  rN  r-   rJ  r‹  r—   r˜   r   r   r  r¢  rC  rË  )rb   r—  r»  r¢   rd   Úmodel_kwargsÚuses_transformers_4_38Úuses_transformers_4_36Útransformers_new_cache_archsÚ
uses_cacheÚ
is_prefillr˜  Úseq_lenrŽ  ÚsizerŸ  ÚbsÚtotal_seq_lenÚattention_mask_2dÚcache_position_Úattention_mask_newÚcacheÚrequires_prompt_injectionr  Únew_past_key_valuesr�  r¡  rá   s                               rh   rÇ  Ú2PeftModelForCausalLM.prepare_inputs_for_generationT  s  € Ø×-Ñ-ˆØ×DÒDÀdÐUÈfÑUˆô
 "+×!2Ñ!2×!8Ñ!8¼×9QÑ9QÓ!RÔV_×VgÑVg×VmÑVmÐnvÓVwÑ!wÐÜ!*×!2Ñ!2×!8Ñ!8¼×9QÑ9QÓ!RÔV_×VgÑVg×VmÑVmÐnvÓVwÑ!wÐÚ'OÐ$Ü×Ñ×"Ñ"¤<×#;Ñ#;Ó<¼y×?PÑ?P×?VÑ?VÐW_Ó?`Ó`à(×/Ñ/°Ô8à+÷ 
Ø"×h t§¡×'=Ñ'=×'HÑ'HÐLhÑ'hð 	ð #×&Ñ&Ð'7Ó8ÀÐD×rÈ<ÐXhÑKiÐjkÑKlÐpqÑKqˆ
à× Ñ ¤H§M¡MÓ1Ø'/˜Ñ$Ø×)×)Ð)Þ˜|×/Ñ/Ð0AÀ4ÓHÑTð #/Ð/@Ñ"A�Ü˜o´´t¨}×=Ñ=Ø-¨aÑ0°Ñ3×9Ñ9¸"Ñ=‘Gà-×<Ñ<Ó>�GØ¨;Ñ7×=Ñ=¸aÑ@Ó@Ø0<¸[Ñ0IÊ!ÈRÉSÈ&Ñ0Q�L Ñ-à".×"2Ñ"2Ð3CÀTÓ"JÐJ�ÒWÜ˜n¬d×3Ñ3ô ˜>Ó*¨aÓ/Ü(ØDÄSÈÓEXÐDYð Zgð góð ô &*¨.×*?Ñ*?Ó*AÓ%BÀ1Ñ%E�Nà# KÑ0×6Ñ6°qÑ9¸;×;YÑ;YÐY�Ü(-¯
ª
°4Ó(8×(;Ñ(;¸LÈÑ<U×<\Ñ<\Ó(]Ð%Ø!×%Ñ%Ó'¨1Ó,ð
 (×-Ñ-¨aÑ0�BØ$9×$?Ñ$?ÀÑ$BÀ^×EYÑEYÐZ[ÑE\Ñ$\�MÜ(-¯
ª
°B¸Ð3FÈn×NbÑNbÑ(cÐ%æ! {×'<Ñ'<ÄX×E[ÑE[Ô]e×]oÑ]oÐDpÓ'pô +0¯,ª,°}È\ÐZeÑMf×MmÑMmÑ*n™ð +7Ð7GÑ*H˜ä)>Ø×+Ñ+Ó-Ø$(Ø'8Ø(4×(8Ñ(8Ð9JÓ(KØ'6Ø#%Ø(5Ø%1×%5Ñ%5°nÀdÓ%Kñ	*Ð&ð 6H�LÐ!1Ò2ô 6;·Y²YÐ@UÐ?fÐlmÑ5n�LÐ!1Ñ2à×Ñ °Ó5ÑAØ×(Ñ(¬X×-CÑ-CÄX×EWÑEWÐ,XÓXà3?ÀÑ3OÐR]×RpÑRpÑ3p�L Ò0ä—M’MØoôð 48�L Ñ0à�z‰zÐ*¨DÓ1Ñ=Ü—’Ønôð ,0�Ð'Ñ(à/;×/?Ñ/?Ð@QÐSWÓ/XˆEà).°$¨÷ )Ü˜5¤,×"4Ñ"4Ó5×T¸e×>RÑ>RÓ>TÔ:Tð &ö )¨[×-BÑ-BÄx×G]ÑG]Ô_g×_qÑ_qÐFrÓ-rä˜e¤\×%7Ñ%7×8Ñ8ÄÈEÔS_×SlÑSl×AmÑAmØ$)×$7Ñ$7‘Mà$&�MØ&*§o¡oØ+¨KÑ8×>Ñ>¸qÑAØ"/ð '6ð 'Ð#ð 3F�Ð.Ò/Þ*Ø $× 4Ñ 4°\À+Ñ5NÓ O�ØŸ/™/°\À+Ñ5N×5TÑ5TÐUVÑ5WÐbj˜/Ðk�Ø!Ÿ*™* ]×%8Ñ%8Ó9�Ü05·	²	¸7ÀMÐ:RÐXYÑ0Z�˜_Ñ-Ø,0�˜[Ñ)ö ˜;×0Ñ0´X×5KÑ5KÌX×M_ÑM_Ð4`Ó`àÐ)Ó*¨k×.LÑ.LÑLÓ*ð Ðð ×"Ñ"Ü×"Ñ"Ü×Ñð+
ó 
ð × Ñ Ð!1°4Ó8ˆAàÐro   )r½  rl  r·  r¸  )r—  rg  )rÓ   rÒ   rm  rn  ro  rU   ro  rÅ  rÁ  rÇ  rs  rt  ru  s   @rh   rL  rL  w  s€   ø† ñ&ðR T]ðfØ$ðfØ3=ðfØMPðfà	÷fð fð ØØØØØ!ØØô]Jò~6%òpð. W[÷ Hõ Hro   rL  c                  ó|   ^ • \ rS rSrSr S
       SU 4S jjjr           SS jrS rSS.SS jjrS	r	U =r
$ )ÚPeftModelForSeq2SeqLMiß  aà  
Peft model for sequence-to-sequence language modeling.

Args:
    model ([`~transformers.PreTrainedModel`]): Base transformer model.
    peft_config ([`PeftConfig`]): Peft config.
    adapter_name (`str`,  *optional*): The name of the adapter, defaults to `"default"`.
    autocast_adapter_dtype (`bool`, *optional*, defaults to `True`):
        Whether to autocast the adapter dtype. Defaults to `True`. Right now, this will only cast adapter weights
        using float16 and bfloat16 to float32, as this is typically required for stable training, and only affect
        select PEFT tuners. If set to `False`, the dtypes will stay the same as those of the corresponding layer.

Example:

    ```py
    >>> from transformers import AutoModelForSeq2SeqLM
    >>> from peft import PeftModelForSeq2SeqLM, get_peft_config

    >>> config = {
    ...     "peft_type": "LORA",
    ...     "task_type": "SEQ_2_SEQ_LM",
    ...     "inference_mode": False,
    ...     "r": 8,
    ...     "target_modules": ["q", "v"],
    ...     "lora_alpha": 32,
    ...     "lora_dropout": 0.1,
    ...     "fan_in_fan_out": False,
    ...     "enable_lora": None,
    ...     "bias": "none",
    ... }

    >>> peft_config = get_peft_config(config)
    >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base")
    >>> peft_model = PeftModelForSeq2SeqLM(model, peft_config)
    >>> peft_model.print_trainable_parameters()
    trainable params: 884736 || all params: 223843584 || trainable%: 0.3952474242013566
    ```
c                ó–   >• [         TU ]  " XU40 UD6  U R                  R                  U l        U R                  R
                  U l        g rk   )rT   rU   r\   rÇ  r½  Ú._prepare_encoder_decoder_kwargs_for_generationÚ8base_model_prepare_encoder_decoder_kwargs_for_generationr¾  s        €rh   rU   ÚPeftModelForSeq2SeqLM.__init__	  s@   ø€ ô 	‰Ò˜¨\ÑD¸VÒDØ8<¿¹×8eÑ8eˆÔ5à�O‰O×JÑJð 	ÕEro   Nc                óÄ
  • U R                   nUR                  (       d�  UR                  [        R                  :X  a  X¼S'   U R
                  " S0 UD6   UR                  5        VVs0 s H  u  pïXàR                  ;  d  M  Xï_M     nnnU R                  " SUUUUUUUUU	U
S.
UD6sS S S 5        $ [        X5      nUb  [        R                  " UUR                  5      R                  UR                  5      nUR                  [        R                  [        R                   4;  a  [        R"                  " UU4SS9nUR%                  SS 5      b_  UR                  [        R&                  [        R(                  4;   a  US   UR                  -   US'   O[*        R,                  " S5        S US'   UR%                  SS 5      b  [*        R,                  " S5        S US'   UR/                  UUUUU	U
S	.5        UR                  [        R&                  [        R(                  4;   a*  U R1                  U5      US
'   U R                  " SUUUS.UD6$ UR                  [        R                  [        R                   4;   aÕ  Uc  U R3                  U5      nUbT  [        R                  " UUR                  5      R                  UR                  5      n[        R"                  " UU4SS9US'   U R1                  US9nUR                  UR4                  5      n[        R"                  " US S 2S UR                  24   U4SS9nU R                  " SUUUS.UD6$ Uc  U R3                  U5      nUcH  UcE  [7        XpR8                  R:                  U R8                  R<                  5      nU R3                  U5      nUbT  [        R                  " UUR                  5      R                  UR                  5      n[        R"                  " UU4SS9US'   Ub{  UR>                  S:X  a  X|S'   OfUR>                  S:X  aV  [        R@                  " UUR                  4S5      R                  UR                  5      n[        R"                  " UU4SS9US'   U R1                  UUS9nUR                  UR4                  5      n[        R"                  " US S 2S UR                  24   U4SS9nUR>                  S:X  a  U R                  " SSU0UD6$ UR>                  S:X  a>  [        R"                  " US S 2UR                  S 24   U4SS9nU R                  " SX6S.UD6$ g s  snnf ! , (       d  f       GN±= f)Nr—  )
r�  rŽ  r�  Údecoder_input_idsÚdecoder_attention_maskÚdecoder_inputs_embedsr�  r‘  r’  r“  r/   rv  r”  r•  r—  rÂ  )rŽ  r  r�  r‘  r’  r“  r˜  )r�  r   r  rŽ  ©r–  )r�  r   r  r�  r·   rÃ  r˜  r�  )r�  r  r	  )!r„  rY   rW   r8   rš  rº  rÀ   rX   r\   r:   rÁ   r›  rO  rL  r  rD  rG  r‹  r¼   rH  rI  r—   r˜   rM  r¢  rC  rü   rD   rR   Úpad_token_idÚdecoder_start_token_idr=  rÆ  )rb   r�  rŽ  r�  r   r  r  r�  r‘  r’  r“  r—  r¢   rd   rã   rä   r–  rŸ  r¡  rÇ  s                       rh   ro  ÚPeftModelForSeq2SeqLM.forward	  s=  € ð ×-Ñ-ˆØ×-×-Ø×$Ñ$¬¯©Ó5Ø%-�zÑ"à×0Ò0Ñ:°6Ó:Ø+1¯<©<¬>Ôeª>¡4 1¸Q×FdÑFdÑ=d›$˜!š$©>�ÑeØ—’ð Ø'Ø#1Ø"/Ø&7Ø+AØ*?Ø!Ø&7Ø)=Ø +ñð ñ÷ ;Ñ:ô  % YÓ>ˆ
Ø!Ñ-ä$)§J¢J¨z¸;×;YÑ;YÓ$Z×$]Ñ$]Ø&×-Ñ-ó%Ð!ð ×$Ñ$¬X×-CÑ-CÄX×EVÑEVÐ,WÓWÜ).¯ªÐ4IÐKaÐ3bÐhiÑ)jÐ&à�:‰:�n dÓ+Ñ7Ø×$Ñ$¬×)?Ñ)?Ä×ASÑASÐ(TÓTà)/°Ñ)?À+×B`ÑB`Ñ)`��~Ò&ä—’ÐuÔvØ)-��~Ñ&Ø�:‰:Ð&¨Ó-Ñ9Ü�MŠMÐtÔuØ'+ˆFÐ#Ñ$Ø�‰à"0Ø*@Ø Ø%6Ø(<Ø*ñô		
ð × Ñ ¤X×%;Ñ%;¼X×=OÑ=OÐ$PÓPà(,¯©¸
Ó(CˆFÐ$Ñ%Ø—?’?ð Ø#Ø"3Ø&;ñð ñ	ð ð ×"Ñ"¤x×'=Ñ'=¼x×?PÑ?PÐ&QÓQØÑ$Ø $× 4Ñ 4°YÓ ?�àÑ)ä(-¯
ª
°:¸{×?]Ñ?]Ó(^×(aÑ(aØ"×)Ñ)ó)Ð%ô ,1¯9ª9Ð6KÈ^Ð5\ÐbcÑ+d�Ð'Ñ(à—o‘o°�oÐ<ˆGØ—j‘j ×!4Ñ!4Ó5ˆGÜ!ŸIšI wªqÐ2R°K×4RÑ4RÐ2RÐ/RÑ'SÐUbÐ&cÐijÑkˆMà—?’?ð Ø+Ø"3Ø&;ñð ñ	ð ð Ñ$Ø $× 4Ñ 4°YÓ ?�Ø$Ñ,Ð1BÑ1JÜ$6ØŸK™K×4Ñ4°d·k±k×6XÑ6Xó%Ð!ð )-×(<Ñ(<Ð=NÓ(OÐ%àÑ)ä(-¯
ª
°:¸{×?]Ñ?]Ó(^×(aÑ(aØ"×)Ñ)ó)Ð%ô ,1¯9ª9Ð6KÈ^Ð5\ÐbcÑ+d�Ð'Ñ(àÑ!Ø×9Ñ9¸QÓ>Ø'-˜8Ò$Ø ×;Ñ;¸qÓ@Ü$)§J¢J°
¸K×<ZÑ<ZÐ/[Ð]aÓ$b×$eÑ$eÐfl×fsÑfsÓ$t�MÜ',§y¢y°-ÀÐ1HÈaÑ'P�F˜8Ñ$Ø—o‘o°Àh�oÐOˆGØ—j‘j ×!4Ñ!4Ó5ˆGÜ!ŸIšI wªqÐ2R°K×4RÑ4RÐ2RÐ/RÑ'SÐUbÐ&cÐijÑkˆMØ×5Ñ5¸Ó:Ø—’ÑM°]ÐMÀfÑMÐMØ×7Ñ7¸1Ó<Ü(-¯	ª	ØšQ × >Ñ >Ñ @Ð@ÑAÐCXÐYÐ_`ñ)Ð%ð —’ð Ø"/ñØ`fñð ð	 =ùóM f÷ ;Ö:ús$   ÁUÁ&U
Á?U
ÂUÕ
UÕ
Uc                ó˜  • U R                   nU R                  U R                  l        U R                  U R                  l         UR                  (       dn  U R
                  " S0 UD6   UR                  5        VVs0 s H  u  p4X0R                  ;  d  M  X4_M     nnnU R                  R                  " S0 UD6nS S S 5        GO~SU;  a  [        S5      eUR                  SS 5      b_  UR                  [        R                  [        R                  4;   a  US   UR                  -   US'   O[         R"                  " S5        S US'   UR                  SS 5      b  [         R"                  " S5        S US'   UR                  [        R                  [        R                  4;   a  U R                  R                  " S0 UD6nGO�UR                  [        R$                  [        R&                  [        R(                  4;   Ga=  [+        U5      nSU;   a  US	 [         R"                  " S5        UR-                  S5      nU R/                  U5      nUR0                  S	   nU R3                  X�R-                  S
S 5      S9n	U	R5                  UR6                  5      n	[8        R:                  " U	S S 2S UR                  24   U4SS9nXqS'   SU;   aX  [8        R<                  " X‚R                  5      R5                  US   R>                  5      n
[8        R:                  " X¡S   4SS9US'   U R                  R                  " S0 UD6$ [@        eU RB                  U R                  l        U RD                  U R                  l        W$ s  snnf ! , (       d  f       NL= f!   U RB                  U R                  l        U RD                  U R                  l        e = f)Nr�  z4input_ids must be provided for Peft model generationr”  r•  r—  rÂ  Úencoder_outputsz[`encoder_outputs` should not be passed to `generate` when using prompt tuning. Ignoring it.r   r—  r˜  r/   rv  r�  rŽ  r	  )#r„  rÇ  r\   rü  rY   rº  rÀ   rX   rÁ  r•   r¼   rW   r8   rH  rI  rO  r—   r˜   rD  rG  rE  r   rË  rC  rB  r¢  rL  rü   rÁ   r‹  r›  r  ÚNotImplementedErrorr½  rý  )rb   r¢   rd   rã   rä   r²  r�  r�  r–  r¡  rŸ  s              rh   rÁ  ÚPeftModelForSeq2SeqLM.generate“	  sY  € Ø×-Ñ-ˆØ8<×8ZÑ8Zˆ�‰Ô5à×?Ñ?ð 	�‰ÔFðC	Ø×1×1Ø×4Ò4Ñ>°vÓ>Ø/5¯|©|¬~Ôiª~¡t qÀ×JhÑJhÑAh›d˜ašd©~�FÑiØ"Ÿo™o×6Ò6Ñ@¸Ñ@�G÷ ?Ñ>ð  fÓ,Ü$Ð%[Ó\Ð\Ø—:‘:˜n¨dÓ3Ñ?Ø"×,Ñ,´×1GÑ1GÌ×I[ÑI[Ð0\Ó\à17¸Ñ1GÈ+×JhÑJhÑ1h˜˜~Ò.ä ŸšØsôð 26˜˜~Ñ.Ø—:‘:Ð.°Ó5ÑAÜ—M’MØrôð 04�FÐ+Ñ,à×(Ñ(¬X×-CÑ-CÄX×EWÑEWÐ,XÓXØ"Ÿo™o×6Ò6Ñ@¸Ñ@’GØ ×*Ñ*Ü×*Ñ*Ü×%Ñ%Ü×4Ñ4ð/ô ô
 & fÓ-�Fà(¨FÓ2Ø"Ð#4Ð5Ü ŸšØyôð !'§
¡
¨;Ó 7�IØ$(×$8Ñ$8¸Ó$C�MØ!.×!4Ñ!4°QÑ!7�JØ"Ÿo™o¸ÏjÉjÐYcÐeiÓNj˜oÐk�GØ%Ÿj™j¨×)<Ñ)<Ó=�Gä$)§I¢I¨w²qÐ:Z¸K×<ZÑ<ZÐ:ZÐ7ZÑ/[Ð]jÐ.kÐqrÑ$s�MØ.;˜?Ñ+à'¨6Ó1Ü05·
²
¸:×GeÑGeÓ0f×0iÑ0iØ"Ð#3Ñ4×;Ñ;ó1Ð-ô 49·9²9Ð>SÐ\lÑUmÐ=nÐtuÑ3v˜Ð/Ñ0àŸ?™?×3Ò3Ñ=°fÑ=Ð=ä-Ð-ð =A×<iÑ<iˆD�O‰OÔ9à×MÑMð �O‰OÔJð ˆNùóA j÷ ?Õ>ûðl	Ø<@×<iÑ<iˆD�O‰OÔ9à×MÑMð �O‰OÔJð úsC   Á#N Á'M?Á;M9ÂM9ÂM?Â9JN Ì;N Í9M?Í?
NÎ	N Î9O	rÜ  c               ó  • U R                   nU R                  " U0 UD6nUR                  [        R                  :X  a  XS'   U$ UR                  [        R
                  [        R                  4;   aœ  UR                  SS 5      nUR                  SS /5      nUR                  S5      nUc5  US L =(       d*    [        U[        5      =(       a    UR                  5       S:H  nU(       a'  US   R                  S   n	U R                  U	5      n
X¥S'   U$ )Nr—  r˜  r�  Úis_first_iterationr   r   )r„  r½  rW   r8   rš  rH  rI  r¼   rr   r   rç  rB  r¢  )rb   r—  r»  r¢   rd   rè  r˜  r�  Úis_prefill_stager–  r÷  s              rh   rÇ  Ú3PeftModelForSeq2SeqLM.prepare_inputs_for_generationÞ	  s  € Ø×-Ñ-ˆØ×DÒDÀdÐUÈfÑUˆØ× Ñ ¤H§M¡MÓ1Ø'/˜Ñ$ð$ Ðð# ×"Ñ"¤x×'=Ñ'=¼x×?QÑ?QÐ&RÓRØ*×.Ñ.Ð/@À$ÓGˆOØ)×-Ñ-Ð.>ÀÀÓGˆNà%Ÿz™zÐ*>Ó?ÐØÑ'ð %¨Ð,÷ hä" ?´EÓ:×fÀ×@^Ñ@^Ó@`ÐdeÑ@eð	 !ö  Ø)Ð*=Ñ>×DÑDÀQÑG�
Ø&*§o¡o°jÓ&AÐ#Ø2EÐ.Ñ/àÐro   )rý  r½  rl  r·  ©NNNNNNNNNNN)r—  re  )rÓ   rÒ   rm  rn  ro  rU   ro  rÁ  rÇ  rs  rt  ru  s   @rh   rú  rú  ß  s|   ø† ñ%ðP T]ð
Ø$ð
Ø3=ð
ØMPð
à	÷
ð 
ð ØØØØ#Ø"ØØØ!ØØôAòFIðV MQ÷ õ ro   rú  c                  ó    ^ • \ rS rSrSr S       S	U 4S jjjr  S
         SU 4S jjjr        SS jr       SS jrSr	U =r
$ )ÚPeftModelForTokenClassificationi÷	  a-  
Peft model for token classification tasks.

Args:
    model ([`~transformers.PreTrainedModel`]): Base transformer model.
    peft_config ([`PeftConfig`]): Peft config.
    adapter_name (`str`,  *optional*): The name of the adapter, defaults to `"default"`.
    autocast_adapter_dtype (`bool`, *optional*, defaults to `True`):
        Whether to autocast the adapter dtype. Defaults to `True`. Right now, this will only cast adapter weights
        using float16 and bfloat16 to float32, as this is typically required for stable training, and only affect
        select PEFT tuners. If set to `False`, the dtypes will stay the same as those of the corresponding layer.

**Attributes**:
    - **config** ([`~transformers.PretrainedConfig`]) -- The configuration object of the base model.
    - **cls_layer_name** (`str`) -- The name of the classification layer.

Example:

    ```py
    >>> from transformers import AutoModelForSequenceClassification
    >>> from peft import PeftModelForTokenClassification, get_peft_config

    >>> config = {
    ...     "peft_type": "PREFIX_TUNING",
    ...     "task_type": "TOKEN_CLS",
    ...     "inference_mode": False,
    ...     "num_virtual_tokens": 20,
    ...     "token_dim": 768,
    ...     "num_transformer_submodules": 1,
    ...     "num_attention_heads": 12,
    ...     "num_layers": 12,
    ...     "encoder_hidden_size": 768,
    ...     "prefix_projection": False,
    ...     "postprocess_past_key_value_function": None,
    ... }

    >>> peft_config = get_peft_config(config)
    >>> model = AutoModelForTokenClassification.from_pretrained("bert-base-cased")
    >>> peft_model = PeftModelForTokenClassification(model, peft_config)
    >>> peft_model.print_trainable_parameters()
    trainable params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117
    ```
c           	     ó   >^• [         TU ]  " XU40 UD6  SS/n[        US5      (       a3  UR                  c  US S  Ul        OUR                  R	                  U5        U R
                  R                  5        H3  u  mn[        U4S jU R                   5       5      (       d  M,  TU l          O   [        U U[        USS 5      UR                  S9  g )Nry  rz  rà  c              3  ó,   >#   • U  H	  oT;   v •  M     g 7frk   r	  r|  s     €rh   r‚   Ú;PeftModelForTokenClassification.__init__.<locals>.<genexpr>1
  ó   øé € ÐOÒ:N¨; $Ö&Ò:Nùr  r€  ©rT   rU   r^   rà  r$  r\   r9  r–   r‚  r=   r_   rÏ   rƒ  s	          @€rh   rU   Ú(PeftModelForTokenClassification.__init__$
  sÃ   ù€ ô 	‰Ò˜¨\ÑD¸VÒDà#/°Ð"9ÐÜ�;Ð 1×2Ñ2Ø×*Ñ*Ñ2Ø.EÁaÐ.H�Õ+à×+Ñ+×2Ñ2Ð3JÔKà—‘×5Ñ5Ö7‰GˆD�!ÜÔO¸$×:NÒ:NÓO×OÓOØ&*�Ô#Ùñ 8ô 	ØØÜ  Ð.?ÀÓFØ&×5Ñ5ó		
ro   c                ó°   >• [        US5      (       a7  SS/nUR                  c  USS Ul        OUR                  R                  U5        [        TU ]  XUS9$ )á’  
Add an adapter to the model based on the passed configuration.

This adapter is not trained. To load a trained adapter, check out [`PeftModel.load_adapter`].

The name for the new adapter should be unique.

The new adapter is not automatically set as the active adapter. Use [`PeftModel.set_adapter`] to set the active
adapter.

Args:
    adapter_name (`str`):
        The name of the adapter to be added.
    peft_config ([`PeftConfig`]):
        The configuration of the adapter to be added.
    low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
        Create empty adapter weights on meta device. Useful to speed up the process when loading saved
        adapters. Don't use this option when creating a new PEFT adapter for training.
    autocast_adapter_dtype (`bool`, *optional*, defaults to `True`):
        Whether to autocast the adapter dtype. Defaults to `True`. Right now, this will only cast adapter
        weights using float16 and bfloat16 to float32, as this is typically required for stable training, and
        only affect select PEFT tuners. If set to `False`, the dtypes will stay the same as those of the
        corresponding layer.

rà  ry  rz  NrK   r‡  rˆ  s         €rh   r]   Ú+PeftModelForTokenClassification.add_adapter=
  sc   ø€ ôB �;Ð 1×2Ñ2Ø'3°WÐ&=Ð#Ø×*Ñ*Ñ2Ø.EÁaÐ.H�Õ+à×+Ñ+×2Ñ2Ð3JÔKä‰wÑ" <ÐPaÐ"ÐbÐbro   c	                óî  • U R                   n
Ub  UOU R                  R                  nU
R                  (       d�  U R                  " S0 U	D6   U	R                  5        VVs0 s H  u  p¼X°R                  ;  d  M  X¼_M     n	nnU
R                  [        R                  :X  a  X‰S'   U R                  " SUUUUUUUS.U	D6sS S S 5        $ [        X5      nUbO  [        R                  " XÚR                  5      R                  UR                   5      n[        R"                  " Xâ4SS9nU	R%                  SS 5      b_  U
R                  [        R&                  [        R(                  4;   a  U	S   U
R                  -   U	S'   O[*        R,                  " S5        S U	S'   U	R/                  UUUUUS.5        U
R                  [        R&                  [        R(                  4;   a  U R0                  " SSU0U	D6$ U	R%                  S	S 5      bv  [        R"                  " [        R2                  " XÚR                  5      R                  U R4                  R6                  R                   5      U	S	   4SS9R9                  5       U	S	'   Uc  U R5                  U5      nU R;                  XØS
9nUR                  UR<                  5      n[        R"                  " Xó4SS9nU R                  " SSU0U	D6$ s  snnf ! , (       d  f       GN.= fr‹  )r„  rR   r™  rY   rº  rÀ   rX   rW   r8   rš  r\   r:   rÁ   r›  rO  rL  r  r‹  r¼   rH  rI  r—   r˜   rM  rœ  r�  rC  rn  rP  r¢  rü   rž  s                   rh   ro  Ú'PeftModelForTokenClassification.forwardg
  s°  € ð ×-Ñ-ˆØ%0Ñ%<‘kÀ$Ç+Á+×B]ÑB]ˆà×-×-Ø×0Ò0Ñ:°6Ó:Ø+1¯<©<¬>Ôeª>¡4 1¸Q×FdÑFdÑ=d›$˜!š$©>�ÑeØ×(Ñ(¬H¯M©MÓ9Ø)1˜:Ñ&Ø—’ð 	Ø'Ø#1Ø"/Ø!Ø&7Ø)=Ø +ñ	ð ñ	÷	 ;Ñ:ô % YÓ>ˆ
ØÑ%ä$)§J¢J¨z×;YÑ;YÓ$Z×$]Ñ$]Ð^l×^sÑ^sÓ$tÐ!Ü"ŸYšYÐ(=Ð'NÐTUÑVˆNØ�:‰:�n dÓ+Ñ7Ø×$Ñ$¬×)?Ñ)?Ä×ASÑASÐ(TÓTà)/°Ñ)?À+×B`ÑB`Ñ)`��~Ò&ä—’ÐuÔvØ)-��~Ñ&Ø�‰à"0Ø Ø%6Ø(<Ø*ñô	
ð × Ñ ¤X×%;Ñ%;¼X×=OÑ=OÐ$PÓPØ×.Ò.ÑM¸ÐMÀfÑMÐMà�z‰zÐ*¨DÓ1Ñ=Ü+0¯9ª9äŸš J×0NÑ0NÓO×RÑRÐSW×SgÑSg×SnÑSn×SuÑSuÓvØÐ/Ñ0ðð ñ,÷ ‘$“&ð Ð'Ñ(ð Ñ$Ø $× 4Ñ 4°YÓ ?�Ø—o‘o°�oÐOˆGØ—j‘j ×!4Ñ!4Ó5ˆGÜ!ŸIšI wÐ&>ÀAÑFˆMØ—?’?ÑI°ÐIÀ&ÑIÐIùói f÷ ;Ö:úr¡  c           
     óH  • [        X5      n	U R                  U	5      n
[        [        R                  " U R
                  R                  5      R                  R                  5       5      nUR                  UUUUUUU
S.5        SU;   a  U R
                  " S
SU0UD6$ U R
                  R                  U R                  5      n[        [        R                  " UR                  5      R                  R                  5       5      nSU;  a  [        S5      eU" S
0 UD6nUS   nS[        U R
                  R                  5       5       VVs/ s H  u  nnUPM
     snn;   a  U R
                  R                  U5      nU R
                  R                  U R                  5      " U5      nS nUb<  [!        5       nU" UR#                  SU R$                  5      UR#                  S5      5      nU(       d  U4USS  -   nUb  U4U-   $ U$ ['        UUUR(                  UR*                  S	9$ s  snnf )Nr£  r˜  r�  r¤  r   r¥  rj  r·   r©  r	  )r:   r¢  rs   rè  ré  r\   ro  r:  r   rM  r?  r<  r•   r9  r¥  r‚  r   r‡  r¯  r#   r¬  r­  )rb   r�  rŽ  r�  r�  r‘  r’  r“  r¢   r–  r˜  r±  r<  r²  Úsequence_outputrÜ   rá   r«  rª  r´  rd  s                        rh   rœ  Ú6PeftModelForTokenClassification._prefix_tuning_forward®
  sð  € ô % YÓ>ˆ
ØŸ/™/¨*Ó5ˆÜœ'×+Ò+¨D¯O©O×,CÑ,CÓD×OÑO×TÑTÓVÓWˆ
Ø�‰à&Ø"0Ø!.Ø%6Ø(<Ø*Ø#2ñô
	
ð  
Ó*Ø—?’?Ñ;¨&Ð;°FÑ;Ð;à(,¯©×(EÑ(EÀd×FdÑFdÓ(eÐ%Üœg×/Ò/Ð0I×0QÑ0QÓR×]Ñ]×bÑbÓdÓeˆJØ ¨
Ó2Ü Ð!oÓpÐpÙ/Ñ9°&Ñ9ˆGØ% a™jˆOØ´°d·o±o×6TÑ6TÓ6VÔ1WÔXÒ1W¡g d¨A›TÑ1WÒXÓXØ"&§/¡/×"9Ñ"9¸/Ó"J�Ø—_‘_×2Ñ2°4×3FÑ3FÔGÈÓXˆFàˆDØÑ!Ü+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR�æØ ˜ W¨Q¨R [Ñ0�Ø-1Ñ-=˜˜ &Ñ(ÐIÀ6ÐIä(ØØØ%×3Ñ3Ø"×-Ñ-ñ	ð ùó Ys   ÅHr¶  )Nr®   r·  rh  ri  r¸  r¹  rº  ru  s   @rh   r  r  ÷	  sÇ   ø† ñ*ðZ [dð
Ø$ð
Ø3=ð
ØTWð
à	÷
ð 
ð: #(Ø'+ð(càð(cð  ð(cð  ð	(cð
 !%ð(cð 
÷(cð (cðX ØØØØØ!ØØôEJðR ØØØØØ!Ø÷4ò 4ro   r  c                  ó¨   ^ • \ rS rSrSr S       S	U 4S jjjr  S
         SU 4S jjjr           SS jr        SS jrSr	U =r
$ )ÚPeftModelForQuestionAnsweringiå
  a¨  
Peft model for extractive question answering.

Args:
    model ([`~transformers.PreTrainedModel`]): Base transformer model.
    peft_config ([`PeftConfig`]): Peft config.
    adapter_name (`str`,  *optional*): The name of the adapter, defaults to `"default"`.
    autocast_adapter_dtype (`bool`, *optional*, defaults to `True`):
        Whether to autocast the adapter dtype. Defaults to `True`. Right now, this will only cast adapter weights
        using float16 and bfloat16 to float32, as this is typically required for stable training, and only affect
        select PEFT tuners. If set to `False`, the dtypes will stay the same as those of the corresponding layer.

**Attributes**:
    - **config** ([`~transformers.PretrainedConfig`]) -- The configuration object of the base model.
    - **cls_layer_name** (`str`) -- The name of the classification layer.

Example:

    ```py
    >>> from transformers import AutoModelForQuestionAnswering
    >>> from peft import PeftModelForQuestionAnswering, get_peft_config

    >>> config = {
    ...     "peft_type": "LORA",
    ...     "task_type": "QUESTION_ANS",
    ...     "inference_mode": False,
    ...     "r": 16,
    ...     "target_modules": ["query", "value"],
    ...     "lora_alpha": 32,
    ...     "lora_dropout": 0.05,
    ...     "fan_in_fan_out": False,
    ...     "bias": "none",
    ... }

    >>> peft_config = get_peft_config(config)
    >>> model = AutoModelForQuestionAnswering.from_pretrained("bert-base-cased")
    >>> peft_model = PeftModelForQuestionAnswering(model, peft_config)
    >>> peft_model.print_trainable_parameters()
    trainable params: 592900 || all params: 108312580 || trainable%: 0.5473971721475013
    ```
c           	     óž  >^• [         TU ]  " XU40 UD6  S/n[        US5      (       a3  UR                  c  US S  Ul        OUR                  R	                  U5        U R
                  R                  5        H3  u  mn[        U4S jU R                   5       5      (       d  M,  TU l          O   [        U U[        USS 5      UR                  S9  g )NÚ
qa_outputsrà  c              3  ó,   >#   • U  H	  oT;   v •  M     g 7frk   r	  r|  s     €rh   r‚   Ú9PeftModelForQuestionAnswering.__init__.<locals>.<genexpr>  r  r  r€  r  )	rb   rc   rd   rO   r¢   Úqa_module_namesrá   rÜ   rg   s	          @€rh   rU   Ú&PeftModelForQuestionAnswering.__init__  s¾   ù€ ô 	‰Ò˜¨\ÑD¸VÒDà'˜.ˆÜ�;Ð 1×2Ñ2Ø×*Ñ*Ñ2Ø.=¹aÐ.@�Õ+à×+Ñ+×2Ñ2°?ÔCà—‘×5Ñ5Ö7‰GˆD�!ÜÔO¸$×:NÒ:NÓO×OÓOØ&*�Ô#Ùñ 8ô 	ØØÜ  Ð.?ÀÓFØ&×5Ñ5ó		
ro   c                ó®   >• [        US5      (       a6  S/nUR                  c  USS Ul        OUR                  R                  U5        [        TU ]  XUS9$ )r  rà  r#  NrK   r‡  )rb   rO   rd   rL   rP   r&  rg   s         €rh   r]   Ú)PeftModelForQuestionAnswering.add_adapter)  s^   ø€ ôB �;Ð 1×2Ñ2Ø+˜nˆOØ×*Ñ*Ñ2Ø.=¹aÐ.@�Õ+à×+Ñ+×2Ñ2°?ÔCä‰wÑ" <ÐPaÐ"ÐbÐbro   c                óú  • U R                   nU
b  U
OU R                  R                  n
UR                  (       dŽ  UR                  [
        R                  :X  a  X¼S'   U R                  " S0 UD6   UR                  5        VVs0 s H  u  pïXàR                  ;  d  M  Xï_M     nnnU R                  " SUUUUUUU	U
S.UD6sS S S 5        $ [        X5      nUbQ  [        R                  " UUR                  5      R                  UR                   5      n[        R"                  " UU4SS9nUR%                  SS 5      b_  UR                  [
        R&                  [
        R(                  4;   a  US   UR                  -   US'   O[*        R,                  " S5        S US'   UR/                  UUUUU	U
S.5        UR                  [
        R&                  [
        R(                  4;   a  U R0                  " SSU0UD6$ UR%                  S	S 5      bw  [        R"                  " [        R2                  " UUR                  5      R                  U R4                  R6                  R                   5      US	   4SS9R9                  5       US	'   Uc  U R5                  U5      nU R;                  US
9nUR                  UR<                  5      n[        R"                  " UU4SS9nU R                  " SSU0UD6$ s  snnf ! , (       d  f       GN3= f)Nr—  )r�  rŽ  r�  Ústart_positionsÚend_positionsr‘  r’  r“  r/   rv  r”  r•  )rŽ  r+  r,  r‘  r’  r“  r�  r—  r  r�  r	  )r„  rR   r™  rY   rW   r8   rš  rº  rÀ   rX   r\   r:   rÁ   r›  rO  rL  r  r‹  r¼   rH  rI  r—   r˜   rM  rœ  r�  rC  rn  rP  r¢  rü   )rb   r�  rŽ  r—  r”  r�  r+  r,  r‘  r’  r“  r—  r¢   rd   rã   rä   r–  rŸ  r¡  s                      rh   ro  Ú%PeftModelForQuestionAnswering.forwardS  s¾  € ð ×-Ñ-ˆØ%0Ñ%<‘kÀ$Ç+Á+×B]ÑB]ˆà×-×-Ø×$Ñ$¬¯©Ó5Ø%-�zÑ"à×0Ò0Ñ:°6Ó:Ø+1¯<©<¬>Ôeª>¡4 1¸Q×FdÑFdÑ=d›$˜!š$©>�ÑeØ—’ð 
Ø'Ø#1Ø"/Ø$3Ø"/Ø&7Ø)=Ø +ñ
ð ñ
÷ ;Ñ:ô % YÓ>ˆ
ØÑ%ä$)§J¢J¨z¸;×;YÑ;YÓ$Z×$]Ñ$]Ð^l×^sÑ^sÓ$tÐ!Ü"ŸYšYÐ(=¸~Ð'NÐTUÑVˆNØ�:‰:�n dÓ+Ñ7Ø×$Ñ$¬×)?Ñ)?Ä×ASÑASÐ(TÓTà)/°Ñ)?À+×B`ÑB`Ñ)`��~Ò&ä—’ÐuÔvØ)-��~Ñ&Ø�‰à"0Ø#2Ø!.Ø%6Ø(<Ø*ñô		
ð × Ñ ¤X×%;Ñ%;¼X×=OÑ=OÐ$PÓPØ×.Ò.ÑM¸ÐMÀfÑMÐMà�z‰zÐ*¨DÓ1Ñ=Ü+0¯9ª9äŸš J°×0NÑ0NÓO×RÑRÐSW×SgÑSg×SnÑSn×SuÑSuÓvØÐ/Ñ0ðð ñ,÷ ‘$“&ð Ð'Ñ(ð Ñ$Ø $× 4Ñ 4°YÓ ?�Ø—o‘o°�oÐ<ˆGØ—j‘j ×!4Ñ!4Ó5ˆGÜ!ŸIšI w°Ð&>ÀAÑFˆMØ—?’?ÑI°ÐIÀ&ÑIÐIùói f÷ ;Ö:ús$   Á-K+ÂK%ÂK%Â K+Ë%K+Ë+
K:c	           
     óä  • [        X5      n
U R                  U
5      n[        [        R                  " U R
                  R                  5      R                  R                  5       5      nU	R                  UUUUUUUS.5        SU;   a  U R
                  " SXES.U	D6$ U R
                  R                  U R                  5      n[        [        R                  " UR                  5      R                  R                  5       5      nSU;  a  [        S5      eU" S0 U	D6nUS   nS[        U R
                  R                  5       5       VVs/ s H  u  nnUPM
     snn;   a  U R
                  R                  U5      nU R
                  R                  U R                  5      " U5      nUR!                  SSS	9u  nnUR#                  S5      R%                  5       nUR#                  S5      R%                  5       nS nUb·  Ub´  ['        UR)                  5       5      S:”  a  UR#                  S5      n['        UR)                  5       5      S:”  a  UR#                  S5      nUR)                  S5      nUR+                  SU5      nUR+                  SU5      n[-        US
9nU" UU5      nU" UU5      nUU-   S-  nU(       d  UU4USS  -   nUb  U4U-   $ U$ [/        UUUUR0                  UR2                  S9$ s  snnf )Nr£  r˜  )r+  r,  r¤  r   r¥  r/   rj  rv  )Úignore_indexr·   )rª  Ústart_logitsÚ
end_logitsr¬  r­  r	  )r:   r¢  rs   rè  ré  r\   ro  r:  r   rM  r?  r<  r•   r9  r¥  r‚  r�  r°  rÊ   rÅ   rï  Úclampr   r!   r¬  r­  )rb   r�  rŽ  r�  r+  r,  r‘  r’  r“  r¢   r–  r˜  r±  r<  r²  r  rÜ   rá   r«  r0  r1  Ú
total_lossÚignored_indexr´  Ú
start_lossÚend_lossrd  s                              rh   rœ  Ú4PeftModelForQuestionAnswering._prefix_tuning_forward   sË  € ô % YÓ>ˆ
ØŸ/™/¨*Ó5ˆÜœ'×+Ò+¨D¯O©O×,CÑ,CÓD×OÑO×TÑTÓVÓWˆ
Ø�‰à&Ø"0Ø!.Ø%6Ø(<Ø*Ø#2ñô
	
ð  
Ó*Ø—?’?Ðj°?ÑjÐciÑjÐjà(,¯©×(EÑ(EÀd×FdÑFdÓ(eÐ%Üœg×/Ò/Ð0I×0QÑ0QÓR×]Ñ]×bÑbÓdÓeˆJØ ¨
Ó2Ü Ð!oÓpÐpÙ/Ñ9°&Ñ9ˆGØ% a™jˆOØ´°d·o±o×6TÑ6TÓ6VÔ1WÔXÒ1W¡g d¨A›TÑ1WÒXÓXØ"&§/¡/×"9Ñ"9¸/Ó"J�Ø—_‘_×2Ñ2°4×3FÑ3FÔGÈÓXˆFØ'-§|¡|°A¸2 |Ð'>Ñ$ˆL˜*Ø'×/Ñ/°Ó3×>Ñ>Ó@ˆLØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆJàˆJØÑ*¨}Ñ/Hä�×+Ñ+Ó-Ó.°Ó2Ø&5×&=Ñ&=¸bÓ&A�OÜ�}×)Ñ)Ó+Ó,¨qÓ0Ø$1×$9Ñ$9¸"Ó$=�Mà ,× 1Ñ 1°!Ó 4�Ø"1×"7Ñ"7¸¸=Ó"I�Ø -× 3Ñ 3°A°}Ó E�ä+¸ÑG�Ù% l°OÓD�
Ù# J°Ó>�Ø(¨8Ñ3°qÑ8�
æØ&¨
Ð3°g¸a¸b°kÑA�Ø3=Ñ3I˜˜¨Ñ.ÐUÈvÐUä/ØØ)Ø%Ø%×3Ñ3Ø"×-Ñ-ñð ùó9 Ys   ÅK,r¶  rl  r·  rh  ri  r  r¸  rº  ru  s   @rh   r!  r!  å
  sÕ   ø† ñ(ðV T]ð
Ø$ð
Ø3=ð
ØMPð
à	÷
ð 
ð: #(Ø'+ð(càð(cð  ð(cð  ð	(cð
 !%ð(cð 
÷(cð (cðX ØØØØØØØØ!ØØôKJð^ ØØØØØØ!Ø÷Eò Ero   r!  c                  óL   ^ • \ rS rSrSrSSU 4S jjjr       SS jrSrU =r$ )	ÚPeftModelForFeatureExtractioniè  aû  
Peft model for extracting features/embeddings from transformer models

Args:
    model ([`~transformers.PreTrainedModel`]): Base transformer model.
    peft_config ([`PeftConfig`]): Peft config.
    adapter_name (`str`,  *optional*): The name of the adapter, defaults to `"default"`.
    autocast_adapter_dtype (`bool`, *optional*, defaults to `True`):
        Whether to autocast the adapter dtype. Defaults to `True`. Right now, this will only cast adapter weights
        using float16 and bfloat16 to float32, as this is typically required for stable training, and only affect
        select PEFT tuners. If set to `False`, the dtypes will stay the same as those of the corresponding layer.

**Attributes**:
    - **config** ([`~transformers.PretrainedConfig`]) -- The configuration object of the base model.

Example:

    ```py
    >>> from transformers import AutoModel
    >>> from peft import PeftModelForFeatureExtraction, get_peft_config

    >>> config = {
    ...     "peft_type": "LORA",
    ...     "task_type": "FEATURE_EXTRACTION",
    ...     "inference_mode": False,
    ...     "r": 16,
    ...     "target_modules": ["query", "value"],
    ...     "lora_alpha": 32,
    ...     "lora_dropout": 0.05,
    ...     "fan_in_fan_out": False,
    ...     "bias": "none",
    ... }
    >>> peft_config = get_peft_config(config)
    >>> model = AutoModel.from_pretrained("bert-base-cased")
    >>> peft_model = PeftModelForFeatureExtraction(model, peft_config)
    >>> peft_model.print_trainable_parameters()
    ```
c                ó*   >• [         TU ]  " XU40 UD6  g rk   )rT   rU   r¾  s        €rh   rU   Ú&PeftModelForFeatureExtraction.__init__  s   ø€ Ü‰Ò˜¨\ÑD¸VÓDro   c                ó&  • U R                   n	U	R                  (       dŒ  U	R                  [        R                  :X  a  XxS'   U R
                  " S0 UD6   UR                  5        V
Vs0 s H  u  p«X R                  ;  d  M  X«_M     nn
nU R                  " SUUUUUUS.UD6sS S S 5        $ [        X5      nUbO  [        R                  " XÉR                  5      R                  UR                  5      n[        R                  " XÒ4SS9nUR!                  SS 5      b_  U	R                  [        R"                  [        R$                  4;   a  US   U	R                  -   US'   O[&        R(                  " S5        S US'   UR!                  SS 5      b  [&        R(                  " S5        S US'   UR+                  UUUUS	.5        U	R                  [        R"                  [        R$                  4;   a(  U R-                  U5      US
'   U R                  " SSU0UD6$ Uc  U R/                  U5      nU R-                  US9nUR                  UR0                  5      n[        R                  " Xã4SS9nU R                  " SSU0UD6$ s  snn
f ! , (       d  f       GNæ= f)Nr—  )r�  rŽ  r�  r‘  r’  r“  r/   rv  r”  r•  r—  rÂ  )rŽ  r‘  r’  r“  r˜  r�  r  r�  r	  )r„  rY   rW   r8   rš  rº  rÀ   rX   r\   r:   rÁ   r›  rO  rL  r  r‹  r¼   rH  rI  r—   r˜   rM  r¢  rC  rü   )rb   r�  rŽ  r�  r‘  r’  r“  r—  r¢   rd   rã   rä   r–  rŸ  r¡  s                  rh   ro  Ú%PeftModelForFeatureExtraction.forward  sY  € ð ×-Ñ-ˆØ×-×-Ø×$Ñ$¬¯©Ó5Ø%-�zÑ"à×0Ò0Ñ:°6Ó:Ø+1¯<©<¬>Ôeª>¡4 1¸Q×FdÑFdÑ=d›$˜!š$©>�ÑeØ—’ð Ø'Ø#1Ø"/Ø&7Ø)=Ø +ñð ñ÷ ;Ñ:ô % YÓ>ˆ
ØÑ%ä$)§J¢J¨z×;YÑ;YÓ$Z×$]Ñ$]Ð^l×^sÑ^sÓ$tÐ!Ü"ŸYšYÐ(=Ð'NÐTUÑVˆNà�:‰:�n dÓ+Ñ7Ø×$Ñ$¬×)?Ñ)?Ä×ASÑASÐ(TÓTà)/°Ñ)?À+×B`ÑB`Ñ)`��~Ò&ä—’ÐuÔvØ)-��~Ñ&Ø�:‰:Ð&¨Ó-Ñ9Ü�MŠMÐtÔuØ'+ˆFÐ#Ñ$Ø�‰à"0Ø%6Ø(<Ø*ñ	ô	
ð × Ñ ¤X×%;Ñ%;¼X×=OÑ=OÐ$PÓPà(,¯©¸
Ó(CˆFÐ$Ñ%Ø—?’?ÑA¨YÐA¸&ÑAÐAàÑ$Ø $× 4Ñ 4°YÓ ?�Ø—o‘o°�oÐ<ˆGØ—j‘j ×!4Ñ!4Ó5ˆGÜ!ŸIšI wÐ&>ÀAÑFˆMØ—?’?ÑI°ÐIÀ&ÑIÐIùó] f÷ ;Ö:ús$   ÁJÁ&I;Á?I;ÂJÉ;JÊ
Jr	  rl  )rc   rb  rd   r1   rO   rt   r¹  )	rÓ   rÒ   rm  rn  ro  rU   ro  rs  rt  ru  s   @rh   r9  r9  è  s8   ø† ñ%÷NEñ Eð
 ØØØØ!ØØ÷?Jò ?Jro   r9  c                  óf   • \ rS rSr% S\S'   S\S'   S\S'   S\S'   S\S	'   S
\S'   S\S'   S\S'   Srg)ÚTunerLayerStatusiU  rt   rÜ   Úmodule_typerP  rÌ  r`  rq   Úmerged_adaptersú&dict[str, bool | Literal['irregular']]r;  Úavailable_adaptersúdict[str, list[str]]Údevicesr	  N©rÓ   rÒ   rm  rn  Ú__annotations__rs  r	  ro   rh   r?  r?  U  s0   ‡ à
ƒIØÓØƒMØÓØÓØ9Ó9Ø!Ó!Ø!Ö!ro   r?  c                óú  • [        U [        5      (       a-  U R                  n[        U[        5      (       d  [	        S5      eOU n/ nUR                  5        GH|  u  p4[        U[        [        45      (       d  M#  [        U[        5      (       a  M:  [        R                  " [        5      nUR                   Hç  n[        XF5      n[        U[        R                  R                   5      (       aO  UR#                  5        H9  u  p‰U	R%                  5        H   n
XX   R'                  U
R(                  5        M"     M;     M†  [        U[        R                  R*                  5      (       a8  UR#                  5        H"  u  pŠXX   R'                  U
R(                  5        M$     Mç  Mé     SS jnUR#                  5        VVs0 s H  u  pŒX‹" U5      _M     nnn[        R                  " [        5      nUR                  UR,                  -    GH  n[        XF5      n[        U[        R                  R                   5      (       a`  UR#                  5        HJ  u  p‰Xè   R/                  U	R%                  5        V
s/ s H  oªR0                  R2                  PM     sn
5        ML     M˜  [        U[        R                  R*                  5      (       d  UR4                  R6                  S:X  d  MÝ  UR#                  5        H,  u  pŠXè   R'                  U
R0                  R2                  5        M.     GM      UR#                  5        VVs0 s H  u  p�U[9        [;        U5      5      _M     nnn[=        U[?        U5      RA                  S5      S   URB                  (       + URD                  URF                  U[9        URI                  5       5      US9nUR'                  U5        GM     U(       d  [K        S5      eU$ s  snnf s  sn
f s  snnf )	a–  Get the status of each adapter layer in the model.

This function returns a list of `TunerLayerStatus` dataclass instances, each of which contains the following
attributes:

- `name` (`str`):
   The name of the adapter layer, e.g. `model.encoder.block.0.layer.0.SelfAttention.q`.
- `module_type` (`str`):
   The type of the adapter layer, e.g. `lora.Linear`.
- `enabled` (`bool`):
   Whether the adapter layer is enabled.
- `active_adapters` (`list[str]`):
   The names of the active adapters, if any, e.g. `["default"]`.
- `merged_adapters` (`list[str]`):
   The names of the merged adapters, if any, e.g. `["default"]`.
- requires_grad : dict[str, bool | Literal["irregular"]]
   The requires_grad status of the parameters for each adapter module. Ideally, it should be either `True` or
   `False`. If the requires_grad status is not consistent across all parameters, the value will be set to
   `"irregular"`.
- `available_adapters` (`list[str]`):
   The names of the available adapters, e.g. `["default"]`.
- `devices` (`dict[str, list[str]]`):
   The devices where the parameters of the given adapter are stored, e.g. `["cuda"]`.

Args:
    model ([Union[`~PeftModel`, `~transformers.PreTrainedModel`, `nn.Module`]]):
        The model to get the adapter layer status from.

Returns:
    list[`peft.peft_model.TunerLayerStatus`]:
        A list of dataclasses, each containing the status of the corresponding adapter layer.

zjget_layer_status() got an invalid PeftModel instance; prefix tuning and adaption prompt are not supported.c                óH   • [        U 5      (       a  g[        U 5      (       d  gg)NTFrÆ  )Úallr–   ©Úvalss    rh   Úcheck_irrgularÚ(get_layer_status.<locals>.check_irrgular¥  s   € Ü�4�y‰yØÜ�t—9‘9ØØro   Ú
BufferDictÚ(r   )rÜ   r@  rÌ  rq   rA  r;  rC  rE  z€No adapter layers found in the model, please ensure that it's a PEFT model or that you have PEFT adapters injected in the model.)rL  z
list[bool]r]  úbool | Literal['irregular'])&rr   rF   r\   r'   r  r  r(   r)   r,   r¾   r¿   rs   Úadapter_layer_namesr_   rÁ   r6  r7  rÀ   r:  rÃ   r;  ÚParameterDictÚother_param_namesr$  r  r  rg   rÓ   rJ  r  r?  ÚreprÚ	partitionÚdisable_adaptersrq   rA  Ú_get_available_adaptersr•   )rc   r\   Úlayer_statusrÜ   r&  Úmapping_requires_grad_listÚadapter_module_nameÚadapter_modulerå   Ú	submodulerR  rM  rL  r;  Ú
devices_ddræ   rE  Ústatuss                     rh   rã  rã  a  sM  € ôD �%œ×#Ñ#Ø×%Ñ%ˆ
Ü˜*¤i×0Ñ0Üðóð ð 1ð ˆ
à+-€LØ"×0Ñ0×2‰ˆÜ˜&¤>Ô3KÐ"L×MÑMÙÜ�fÔ4×5Ñ5ñ ô =H×<SÒ<SÔTXÓ<YÐ"Ø#)×#=Ô#=ÐÜ$ VÓAˆNÜ˜.¬%¯(©(×*=Ñ*=×>Ñ>Ø&4×&:Ñ&:Ö&<‘N�CØ!*×!5Ñ!5Ö!7˜Ø2Ñ7×>Ñ>¸u×?RÑ?RÖSó "8ó '=ô ˜N¬E¯H©H×,BÑ,B×CÑCØ"0×"6Ñ"6Ö"8‘J�CØ.Ñ3×:Ñ:¸5×;NÑ;NÖOó #9ñ ñ $>ô	ð E_×DdÑDdÔDfÔgÒDf±y°s˜˜n¨TÓ2Ò2ÑDfˆÑgä ×,Ò,¬TÓ2ˆ
Ø#)×#=Ñ#=À×@XÑ@XÕ#XÐÜ$ VÓAˆNÜ˜.¬%¯(©(×*=Ñ*=×>Ñ>Ø&4×&:Ñ&:Ö&<‘N�CØ‘O×*Ñ*È9×K_ÑK_ÔKaÓ+bÒKaÀ%¯L©L×,=Ô,=ÑKaÑ+bÖcó '=ä˜N¬E¯H©H×,BÑ,B×CÑCØ×(Ñ(×1Ñ1°\ÕAà"0×"6Ñ"6Ö"8‘J�CØ‘O×*Ñ*¨5¯<©<×+<Ñ+<Ö=ô #9ñ $Yð :D×9IÑ9IÔ9KÔLÒ9K©X¨S�3œœs 3›xÓ(Ò(Ñ9KˆÑLä!ØÜ˜V›×.Ñ.¨sÓ3°AÑ6Ø×/Ñ/Ô/Ø"×2Ñ2Ø"×2Ñ2Ø'Ü% f×&DÑ&DÓ&FÓGØñ	
ˆð 	×Ñ˜F×#ño 3ör Üð%ó
ð 	
ð
 ÐùóC hùò ,cùó Ms   Æ:O,É0O2Ì9!O7c                  óŽ   • \ rS rSr% S\S'   S\S'   S\S'   S\S'   S\S	'   S\S
'   S\S'   S\S'   S\S'   S\S'   S\S'   S\S'   Srg)rk  iÐ  rt   Úbase_model_typeÚadapter_model_typezdict[str, str]Ú
peft_typesrf  r«  Útotal_paramsÚnum_adapter_layersrQ  rÌ  z list[str] | Literal['irregular']rq   rA  rB  r;  r`  rC  rD  rE  r	  NrF  r	  ro   rh   rk  rk  Ð  sJ   ‡ àÓØÓØÓØÓØÓØÓØ(Ó(Ø5Ó5Ø5Ó5Ø9Ó9Ø!Ó!Ø!Ö!ro   rk  c                óê  • [        U [        5      (       a×  [        U R                  [        5      (       d  [	        S5      eU R                  5       R                  R                  nU R                  5       u  p#U R                  nUR                  R                  5        VVs0 s H-  u  pVU[        UR                  5      R                  S5      S   _M/     nnnUR                  R                  nOh[        U [        5      (       a4  U R                  R                  n[        R                  U 5      u  p#U n0 nSnOSn[        R                  U 5      u  p#U n0 nSn[        U 5      n	[!        U	5      n
U	 Vs1 s H  o»R"                  iM     nn[!        U5      S:X  a  UR%                  5       nOSn['        [)        5       R*                  " S U	 5       6 5      nU	 Vs1 s H  n[-        UR.                  5      iM     nnU(       d  / nO+[!        U5      S:X  a  [1        UR%                  5       5      nOSn[)        5       nU	 H  nUR3                  UR4                  5        M      ['        U5      nU	 H<  n[)        UR6                  5      [)        UR4                  5      -
  nUU-  (       d  M:  Sn  O   [8        R:                  " [0        5      nU	 H;  nUR<                  R                  5        H  u  nnUU   R?                  U5        M     M=     SS	 jnUR                  5        VVs0 s H  u  nnUU" U5      _M     nnn[8        R:                  " [0        5      nU	 H;  nUR@                  R                  5        H  u  nnUU   RC                  U5        M     M=     UR                  5        VVs0 s H  u  nnU['        [)        U5      5      _M     nnn[E        UUUUUU
UUUUUUS
9nU$ s  snnf s  snf s  snf s  snnf s  snnf )a¶  Get the status of tuners of the model.

This function returns a `TunerModelStatus` dataclass instance, which contains the following attributes:

- `base_model_type` (`str`):
   The type of the base model, e.g. `T5Model`.
- `adapter_model_type` (`str`):
   The type of the adapter model, e.g. `LoraModel`.
- `peft_types` (`dict[str, str]`):
   The mapping of adapter name to adapter type, e.g. `{"default": "LORA"}`.
- `trainable_params` (`int`):
   The number of trainable parameters in the model.
- `total_params` (`int`):
   The total number of parameters in the model.
- `num_adapter_layers` (`int`):
   The number of adapter layers in the model.
- `enabled` (`bool`, `Literal["irregular"]`):
   Whether all adapter layers are enabled. If some are enabled and some are not, this will be `"irregular"`. This
   means that your model is in an inconsistent state and might not work as expected.
- `active_adapters` (`list[str]`, `Literal["irregular"]`):
   The names of the active adapters. If the active adapters are not consistent across all layers, this will be
   `"irregular"`, which means that your model is in an inconsistent state and might not work as expected.
- `merged_adapters` (`list[str]`, `Literal["irregular"]`):
   The names of the merged adapters. If the merged adapters are not consistent across all layers, this will be
   `"irregular"`, which means that your model is in an inconsistent state and might not work as expected.
- `requires_grad` (`dict[str, bool | Literal["irregular"]]`):
   Whether for the given adapter, all adapter layers have `requires_grad` set to `True` or `False`. If there is a
   mix, this will be set to `"irregular"`, which means that your model is in an inconsistent state and might not
   work as expected.
- `available_adapters` (`list[str]`):
   The names of the available adapters, e.g. `["default"]`.
- `devices` (`dict[str, list[str]]`):
   The devices where the parameters of the given adapter are stored, e.g. `["cuda"]`.

Args:
    model ([Union[`~PeftModel`, `~transformers.PreTrainedModel`, `nn.Module`]]):
        The model to get the adapter layer status from.

Returns:
    `peft.peft_model.TunerModelStatus`:
        A dataclass containing the status of the model.

zjget_model_status() got an invalid PeftModel instance; prefix tuning and adaption prompt are not supported.r„   rj  r^  Úotherr/   rÆ  c              3  ó8   #   • U  H  oR                   v •  M     g 7frk   )rC  )r€   r_  s     rh   r‚   Ú#get_model_status.<locals>.<genexpr>.  s   é € Ð8nÒamÐW]×9RÖ9RÒamùs   ‚c                ód   • [        S U  5       5      (       a  g[        S U  5       5      (       a  gg)Nc              3  ó(   #   • U  H  oS L v •  M
     g7f)TNr	  ©r€   ræ   s     rh   r‚   Ú;get_model_status.<locals>.check_irrgular.<locals>.<genexpr>U  s   é € Ð+¢d˜s�d�{¢dùó   ‚Tc              3  ó(   #   • U  H  oS L v •  M
     g7f)FNr	  rl  s     rh   r‚   rm  W  s   é € Ð,¢t �e�|¢tùrn  FrÆ  )rJ  rK  s    rh   rM  Ú(get_model_status.<locals>.check_irrgularT  s-   € ÜÑ+¡dÓ+×+Ñ+ØÜÑ,¡tÓ,×,Ñ,ØØro   )ra  rb  rc  r«  rd  re  rÌ  rq   rA  r;  rC  rE  )rL  z!list[bool | Literal['irregular']]r]  rQ  )#rr   rF   r\   r'   r  rJ  rg   rÓ   r¯  rd   rÀ   rt   rW   rV  r    rã  rÅ   rÌ  rË  rJ  r  rI  ræ  rq   rs   rM  rA  rC  r¾   r¿   r;  rÃ   rE  r$  rk  )rc   ra  r«  rd  r\   rå   rR   rc  rb  rY  re  r_  Úenabled_setrÌ  rC  Úall_active_adaptersrq   Ú
merged_allrA  ÚunmergedÚrequires_grad_allræ   rM  rL  r;  r^  rE  Úadapter_model_statuss                               rh   rË  rË  à  s¶  € ôX �%œ×#Ñ#Ü˜%×*Ñ*¬I×6Ñ6Üðóð ð  ×.Ñ.Ó0×:Ñ:×CÑCˆØ).×)JÑ)JÓ)LÑ&ÐØ×%Ñ%ˆ
ØWa×WmÑWm×WsÑWsÔWuÔvÒWuÉÈ�cœ3˜v×/Ñ/Ó0×:Ñ:¸3Ó?ÀÑCÒCÑWuˆ
ÑvØ'×1Ñ1×:Ñ:ÑÜ	�Eœ?×	+Ñ	+ØŸ/™/×2Ñ2ˆÜ)2×)NÑ)NÈuÓ)UÑ&ÐØˆ
Øˆ
Ø#Ñà!ˆÜ)2×)NÑ)NÈuÓ)UÑ&ÐØˆ
Øˆ
Ø#Ðä# EÓ*€LÜ˜\Ó*Ðá;GÓHº<°Ÿnœn¹<€KÐHä
ˆ;Ó˜1ÓØ—/‘/Ó#‰àˆä$*¬3«5¯;ª;Ñ8nÑamÓ8nÐ+oÓ$pÐñ ^jÓ0jÒ]iÐSY´°v×7MÑ7MÖ1NÑ]iÐÐ0jæØ‰Ü	Ð Ó	! QÓ	&ÜÐ2×6Ñ6Ó8Ó9‰à%ˆô ›5€JÛˆØ×Ñ˜&×0Ñ0Ö1ñ ô 9?¸zÓ8J€OÛˆÜ�v×0Ñ0Ó1´C¸×8NÑ8NÓ4OÑOˆØ�j× Ñ à)ˆOÙñ ô GR×F]ÒF]Ô^bÓFcÐÛˆØ×,Ñ,×2Ñ2Ö4‰HˆC�Ø˜cÑ"×)Ñ)¨#Ö.ó 5ñ ô
ð AR×@WÑ@WÔ@YÔZÒ@Y±9°3¸�S™.¨Ó.Ò.Ñ@Y€MÑZä×(Ò(¬Ó.€JÛˆØŸ™×,Ñ,Ö.‰HˆC�Ø�s‰O×"Ñ" 3Ö'ó /ñ ð 6@×5EÑ5EÔ5GÔHÒ5G©¨¨cˆs”Fœ3˜s›8Ó$Ò$Ñ5G€GÑHä+Ø'Ø-ØØ)Ø!Ø-ØØ'Ø'Ø#Ø-ØñÐð  Ðùóy wùò$ Iùò 1kùóT [ùó Is   Â4OÅ/OÇO$ÌO)Î"O/)rc   rb  r]  rj  )rc   rb  r]  rk  )pÚ
__future__r   r¾   rÇ   rè  r™   r—   Úcollections.abcr   Ú
contextlibr   r   r   Údataclassesr   Útypingr	   r
   r   r   Úpackaging.versionrŽ  rÁ   r‘  Ú
accelerater   r   Úaccelerate.hooksr   r   r   Úaccelerate.utilsr   r   Úhuggingface_hubr   r   r   r   Úsafetensorsr   Úsafetensors.torchr   rË   Útorch.nnr   r   r   r   r   r   r    Útransformers.modeling_outputsr!   r"   r#   Útransformers.utilsr$   Úpeft.tuners.lora.variantsr%   r&   Úpeft.tuners.tuners_utilsr'   r(   Ú
peft.utilsr)   Úpeft.utils.constantsr*   Úpeft.utils.integrationsr+   Úpeft.utils.otherr,   r-   r.   rú   r0   rR   r1   r"  r2   r3   r4   Úutilsr5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   r6  ÚModulerF   rw  rL  rú  r  r!  r9  r?  rã  rk  rË  r	  ro   rh   Ú<module>rŽ     sO  ðõ #ã Û Û Û 	Û Ý $ß 2Ý Ý !ß 0Ó 0ã Û Û ß <ß ^Ñ ^ß Fß SÓ SÝ !Ý 9ß AÑ Aß RÓ Rß wÑ wÝ -ç cß >Ý /Ý 3Ý 6ß lÑ lå Ý ß iÑ i÷÷ ÷ ÷ ó ô(j:� §¡§¡ô j:ôZ1¨ô ôDe˜9ô eôPU˜Iô Uôpk iô kô\@ Iô @ôFjJ Iô jJðZ ÷"ð "ó ð"ôlð^ ÷"ð "ó ð"õQ ro   