ó
    >:jÀ  ã                  óÐ  • S SK Jr  S SKrS SKrS SKrS SKrS SKJr  S SKJ	r	  S SK
JrJr  S SKJrJr  S SKrS SKrS SKrS SKJr  S SKJrJrJr  S S	KJrJrJrJrJr  S S
KJ r J!r!J"r"J#r#J$r$J%r%J&r&  S SK'J(r(  S SK)J*r*J+r+J,r,J-r-J.r.  S SK/J0r0  SSK1J2r2  SSK3J4r4  SSK5J6r6  SSK7J8r8  SSK9J:r:  SSK;J<r<  SSK=J>r>  SSK?J@r@JArAJBrBJCrC  SSKDJErE  SSKFJGrG  SSKHJIrI  S rJS rKSS jrL " S S\5      rMg) é    )ÚannotationsN)Úcontextmanager)Úreplace)ÚpartialÚreduce)ÚLiteralÚOptional)Únn)Úis_bnb_4bit_availableÚis_bnb_availableÚis_transformers_ge_v5_4_0)Ú	BaseTunerÚBaseTunerLayerÚfind_parameter_name_by_moduleÚget_device_mapÚreplicate_layers)Ú2TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPINGÚAuxiliaryTrainingWrapperÚModulesToSaveWrapperÚ_freeze_adapterÚ_get_submodulesÚget_peft_model_state_dictÚget_quantization_config)ÚTpInfo)Údare_linearÚ	dare_tiesÚmagnitude_pruneÚtask_arithmeticÚties)Úget_pattern_keyé   )Údispatch_aqlm)Údispatch_awq)Ú
LoraConfig)Údispatch_eetq)Údispatch_gptq)Údispatch_hqq)Údispatch_inc)ÚConv2dÚ	LoraLayerÚParamWrapperÚdispatch_default)Údispatch_transformer_engine)Údispatch_torchao)Údispatch_megatronc                ó   • X2S'   X4$ )NÚadapter_names© )ÚtargetÚargsÚkwargsr1   s       ÚS/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/lora/model.pyÚ_adapter_names_pre_forward_hookr7   @   s   € à+ˆ?ÑØˆ<Ðó    c                ó   • X2S'   X4$ )NÚalora_offsetsr2   )r3   r4   r5   r:   s       r6   Ú_alora_offsets_pre_forward_hookr;   F   s   € Ø+ˆ?ÑØˆ<Ðr8   c                óT   • [        U S5      (       d  gU R                  5       nXL a  gU$ )zSCheck if the model has an encoder and if it has, returns it; otherwise returns NoneÚget_encoderN)Úhasattrr=   )ÚmodelÚencoders     r6   Ú_get_encoderrA   K   s0   € ä�5˜-×(Ñ(Øà×ÑÓ!€Gð ÒØØ€Nr8   c                  ó*  ^ • \ rS rSr% SrSrS\S'   \r\	r
SS jrSS.   SS	 jjrS
 r\S 5       r\S 5       rU 4S jrS r        SS jr       S                     SS jjr   SS jrS rSSS jjrSS jrSS jrSrU =r$ ) Ú	LoraModeléX   a
  
Creates Low Rank Adapter (LoRA) model from a pretrained transformers model.

The method is described in detail in https://huggingface.co/papers/2106.09685.

Args:
    model ([`torch.nn.Module`]): The model to be adapted.
    config ([`LoraConfig`]): The configuration of the Lora model.
    adapter_name (`str`): The name of the adapter, defaults to `"default"`.
    low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
        Create empty adapter weights on meta device. Useful to speed up the loading process.

Returns:
    `torch.nn.Module`: The Lora model.

Example:

    ```py
    >>> from transformers import AutoModelForSeq2SeqLM
    >>> from peft import LoraModel, LoraConfig

    >>> config = LoraConfig(
    ...     task_type="SEQ_2_SEQ_LM",
    ...     r=8,
    ...     lora_alpha=32,
    ...     target_modules=["q", "v"],
    ...     lora_dropout=0.01,
    ... )

    >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base")
    >>> lora_model = LoraModel(model, config, "default")
    ```

    ```py
    >>> import torch
    >>> import transformers
    >>> from peft import LoraConfig, PeftModel, get_peft_model, prepare_model_for_kbit_training

    >>> rank = ...
    >>> target_modules = ["q_proj", "k_proj", "v_proj", "out_proj", "fc_in", "fc_out", "wte"]
    >>> config = LoraConfig(
    ...     r=4, lora_alpha=16, target_modules=target_modules, lora_dropout=0.1, bias="none", task_type="CAUSAL_LM"
    ... )
    >>> quantization_config = transformers.BitsAndBytesConfig(load_in_8bit=True)

    >>> tokenizer = transformers.AutoTokenizer.from_pretrained(
    ...     "kakaobrain/kogpt",
    ...     revision="KoGPT6B-ryan1.5b-float16",  # or float32 version: revision=KoGPT6B-ryan1.5b
    ...     bos_token="[BOS]",
    ...     eos_token="[EOS]",
    ...     unk_token="[UNK]",
    ...     pad_token="[PAD]",
    ...     mask_token="[MASK]",
    ... )
    >>> model = transformers.GPTJForCausalLM.from_pretrained(
    ...     "kakaobrain/kogpt",
    ...     revision="KoGPT6B-ryan1.5b-float16",  # or float32 version: revision=KoGPT6B-ryan1.5b
    ...     pad_token_id=tokenizer.eos_token_id,
    ...     use_cache=False,
    ...     device_map={"": rank},
    ...     torch_dtype=torch.float16,
    ...     quantization_config=quantization_config,
    ... )
    >>> model = prepare_model_for_kbit_training(model)
    >>> lora_model = get_peft_model(model, config)
    ```

**Attributes**:
    - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
    - **peft_config** ([`LoraConfig`]): The configuration of the Lora model.
Úlora_ÚstrÚprefixc                óR   • UR                   (       a  [        X!R                   5        gg)zß
A private method to modify the model structure before adapter is applied.

Args:
    peft_config (`PeftConfig`):
        The prepared adapter config.
    model (`nn.Module`):
        The model that is going to be adapted.
N)Úlayer_replicationr   )ÚselfÚpeft_configr?   s      r6   Ú_prepare_modelÚLoraModel._prepare_model¥   s    € ð ×(×(Ü˜U×$AÑ$AÕBð )r8   N)Úparameter_namec          	     óœ	  ^• Uc  [        S5      eUR                  (       aM  [        U4S jU R                  R	                  5        5       5      nU(       a  [        SUR                   S35      e[        UR                  R                  5       U5      n	[        UR                  R                  5       U5      n
UR                  R                  X‘R                  5      nUR                  R                  X¡R                  5      nU[        US/ 5      =(       d    / ;   n0 nU(       aY  U R                  R                  5       nUR                  T   nUR                   T   nUR#                  5       UR#                  5       S.nUUU[        U R                  SS5      [        U R                  S	S5      UR$                  R&                  UUS
.n [(        R*                  " S5      " U R                  5      US'   / SQnU H$  n[/        U R                  US9nUc  M  UUU S3'   M&     SSKJn  [5        U[6        5      =(       a    TUR8                  ;   n[5        U[:        5      (       a>  [5        UU5      (       d-  U(       d&  UR=                  TUUUUS9  UR?                  5       nUnO“[5        U[6        5      (       a  XsR@                  :X  a  [        S5      e[C        U R                  5      nU RD                  " UTU4SU0UD6nTU RF                  ;  a  URI                  S5        U RK                  XTUU5        UnUn[        USS 5      n[        USS 5      nUGbp  [L        (       d  [O        S5      eSSK(J)n  SnUU;  a  [T        RV                  " SU S35        g / n / n![X        RZ                  " SSU5      n"US;   a‹  US:X  a/  U R]                  U" S T S!35        UR^                  T   n#U S T 34n$O.U R]                  U" S"T S!35        UR8                  T   n#U S"T 34n$U!R]                  U5        U" U R                  U#UU$U5        OLU R]                  U" S#35        U R]                  U" S$T 35        U!R]                  U5        U!R]                  S%5        [a        [c        [e        U U!5      5      UU R                  Rf                  S&9Ul4        g g ! [,         a     GNÜf = f)'NzCurrent Key shouldn't be `None`c              3  óP   >#   • U  H  u  pUT:w  d  M  UR                   v •  M     g 7f©N)Útarget_parameters)Ú.0ÚkeyÚconfÚadapter_names      €r6   Ú	<genexpr>Ú0LoraModel._create_and_replace.<locals>.<genexpr>Â   s)   øé € ð 2Ú8P©9¨3ÐTWÐ[gÑTgÓ&�×&Ö&Ò8Pùs   ƒ&“&z-Adding a LoRA config with `target_parameters=z«` but there are already other LoRA adapters on this model that use `target_parameters`. At the moment, only one LoRA adapter per model with `target_parameters` is allowed.Útarget_modules_to_tie)Úlora_AÚlora_BÚis_loaded_in_8bitFÚis_loaded_in_4bit)ÚrÚ
lora_alphaÚtarget_nameÚloaded_in_8bitÚloaded_in_4bitÚephemeral_gpu_offloadrN   Útied_adapterz:hf_quantizer.quantization_config.get_apply_tensor_subclassÚget_apply_tensor_subclass)ÚgptqÚaqlmÚawq)ÚmethodÚ_quantization_configr   )ÚAdaLoraLayer)r_   r`   Úconfigz—Trying to target the same nn.Parameter twice, this should not happen. Please open an issue on the PEFT repo: https://github.com/huggingface/peft/issuesÚ
device_mapÚ_hf_tp_planÚ_hf_device_meshz«The base model is tensor-parallel sharded but the installed version of Transformers does not support LoRA with Tensor Parallelism. Please upgrade to transformers >= 5.4.0.)Ú#add_tensor_parallel_hooks_to_module)ÚcolwiseÚrowwiseÚembedding_rowwisez	TP plan "zk" on the base layer is not supported for LoRA. LoRA adapters will be created without tensor parallel hooks.z\d+Ú*)rq   rr   rq   z.lora_B.z.weightz.lora_A.z.base_layer.weightz.lora_embedding_A.Úembedding_colwise)Útp_planÚdevice_meshÚtp_size)5Ú
ValueErrorrR   ÚanyrK   Úitemsr    Úrank_patternÚkeysÚalpha_patternÚgetr^   r_   Úgetattrr?   Úget_input_embeddingsÚlora_embedding_AÚlora_embedding_BÚtÚruntime_configrc   ÚoperatorÚ
attrgetterÚAttributeErrorr   Úpeft.tuners.adalorark   Ú
isinstancer+   rZ   r*   Úupdate_layerÚget_base_layerrN   r   Ú_create_new_moduleÚactive_adaptersÚrequires_grad_Ú_replace_moduler   ÚRuntimeErrorÚ)transformers.integrations.tensor_parallelrp   ÚwarningsÚwarnÚreÚsubÚappendr[   r   ÚdictÚzipÚ_tp_sizeÚ_tp_info)%rJ   Úlora_configrV   r3   r`   ÚparentÚcurrent_keyrN   Úother_configs_use_target_paramsÚr_keyÚ	alpha_keyr^   ÚalphaÚis_tiedrd   Útied_moduleÚemb_AÚemb_Br5   Úquant_methodsÚquant_methodÚquantization_configrk   Úwrap_target_paramÚ
base_layerÚlora_modulerm   Ú
new_modulerv   rw   rp   Ú_SUPPORTED_TP_PLANSÚtp_plan_keysÚtp_plansÚgeneric_keyÚ	tp_moduleÚtp_layer_names%     `                                  r6   Ú_create_and_replaceÚLoraModel._create_and_replace²   sã  ø€ ð ÑÜÐ>Ó?Ð?à×(×(ä.1ô 2Ø8<×8HÑ8H×8NÑ8NÔ8Pó2ó /Ð+ö /Ü ØCÀK×DaÑDaÐCbð cVð Vóð ô   × 8Ñ 8× =Ñ =Ó ?ÀÓMˆÜ# K×$=Ñ$=×$BÑ$BÓ$DÀkÓRˆ	Ø×$Ñ$×(Ñ(¨·±Ó>ˆØ×)Ñ)×-Ñ-¨i×9OÑ9OÓPˆð ¤'¨+Ð7NÐPRÓ"S×"YÐWYÑZˆØˆÞØŸ*™*×9Ñ9Ó;ˆKØ×0Ñ0°Ñ>ˆEØ×0Ñ0°Ñ>ˆEà&+§g¡g£i¸5¿7¹7»9ÑEˆLð ØØ&Ü% d§j¡jÐ2EÀuÓMÜ% d§j¡jÐ2EÀuÓMØ%0×%?Ñ%?×%UÑ%UØ,Ø(ñ	
ˆð	Ü2:×2EÒ2EØLô3à�j‰jó3ˆFÐ.Ñ/ò 0ˆÛ)ˆLÜ"9¸$¿*¹*È\Ñ"ZÐØ"Ó.Ø@S�˜,˜Ð';Ð<Ó=ñ *õ 	5ô ' v¬|Ó<×`À,ÐRX×R_ÑR_ÑB_ÐÜ�fœi×(Ñ(´¸FÀL×1QÑ1QÖZkØ×ÑØØØ Ø'Ø"ð  ñ ð  ×.Ñ.Ó0ˆJØ ‰Kä˜&¤,×/Ñ/°^×G\ÑG\Ó5\Ü ðLóð ô (¨¯
©
Ó3ˆJØ×0Ò0°¸lÈFÑtÐ_iÐtÐmsÑtˆJØ 4×#7Ñ#7Ó7à×)Ñ)¨%Ô0Ø× Ñ  °jÀ&ÔIàˆJØ$ˆKä˜* m°TÓ:ˆÜ˜jÐ*;¸TÓBˆð Òß,Ò,Ü"ðeóð õð #NÐàÐ1Ó1Ü—’Ø ˜yð )Sð Sõð
  "�Ø�Ü Ÿfšf V¨S°+Ó>�ØÐ4Ó4Ø )Ó+Ø$×+Ñ+¨{¨m¸8ÀLÀ>ÐQXÐ,YÔZØ$/×$6Ñ$6°|Ñ$D˜	Ø,7¨=¸ÀÀÐ)OÐ(Q™à$×+Ñ+¨{¨m¸8ÀLÀ>ÐQXÐ,YÔZØ$/×$6Ñ$6°|Ñ$D˜	Ø,7¨=¸ÀÀÐ)OÐ(Q˜Ø—O‘O GÔ,Ù7ØŸ
™
Ø!ØØ%Ø#õð !×'Ñ'¨;¨-Ð7IÐ(JÔKØ ×'Ñ'¨;¨-Ð7IÈ,ÈÐ(XÔYØ—O‘O GÔ,ð —O‘OÐ$7Ô8ä'-Ü ¤ \°8Ó!<Ó=Ø +Ø ŸJ™J×/Ñ/ñ(�Õ$ða øô[ ó 	Úð	ús   Ç)R= Ò=
SÓ
Sc                óŒ  ^• [        XU5        [        US5      (       a  UR                  n[        R                  " S5      mUR                  5        Hð  u  pVU R                  U;   d  SU;   d  M  [        US5      (       a  UR                  nOp[        US5      (       a  UR                  nOR[        US5      (       a  UR                  nO4[        USS 5      b  UR                  nO[        UR                  5       5      n[        U4S jUR                  5        5       5      (       a  MÕ  UR                  UR                  5        Mò     g )	Nr«   ÚmetaÚranknumÚqweightÚW_qÚweightÚin_proj_weightc              3  ó@   >#   • U  H  oR                   T:H  v •  M     g 7frQ   )Údevice)rS   Úpr·   s     €r6   rW   Ú,LoraModel._replace_module.<locals>.<genexpr>i  s   øé € ÐIÒ5H°Ÿ8™8 tÖ+Ò5Hùó   ƒ)Úsetattrr>   r«   Útorchr¾   Únamed_modulesrG   r¹   rº   r»   r€   r¼   ÚnextÚ
parametersrz   Úto)	rJ   r�   Ú
child_namer­   ÚchildÚnameÚmoduler»   r·   s	           @r6   r�   ÚLoraModel._replace_moduleP  sù   ø€ ô 	� JÔ/ô
 �5˜,×'Ñ'Ø×$Ñ$ˆEä�|Š|˜FÓ#ˆà&×4Ñ4Ö6‰LˆDØ—‘˜tÓ#¨°dÕ):Ü˜5 )×,Ñ,Ø"Ÿ]™]‘FÜ˜U E×*Ñ*Ø"ŸY™Y‘FÜ˜U H×-Ñ-Ø"Ÿ\™\‘FÜ˜UÐ$4°dÓ;ÑGØ"×1Ñ1‘Fä! %×"2Ñ"2Ó"4Ó5�FÜÔI°V×5FÑ5FÔ5HÓI×IÓIØ—I‘I˜fŸm™mÖ,ò 7r8   c                óÌ  • / nU R                   (       a  S nUR                  U5        [        5       (       a  SSKJn  UR                  U5        [        5       (       a  SSKJn  UR                  U5        UR                  [        [        [        [        [        [        [        [        [         ["        /
5        S nU H  n	U	" X!4SU 0UD6nUc  M    O   Uc  [%        SU S35      eU$ )Nc                óÜ   • S n[        U [        5      (       a  U R                  5       nOU nUR                  R	                  5        H#  u  pg[        XV5      (       d  M  U" X4SU0UD6n  U$    U$ )Nrl   )rŠ   r   rŒ   Ú_custom_modulesr{   )r3   rV   rl   r5   r­   Útarget_base_layerrT   Ú
custom_clss           r6   Údynamic_dispatch_funcÚ;LoraModel._create_new_module.<locals>.dynamic_dispatch_funcu  sw   € Ø!�
ä˜f¤n×5Ñ5Ø(.×(=Ñ(=Ó(?Ñ%à(.Ð%à'-×'=Ñ'=×'CÑ'CÖ'E‘O�CÜ!Ð"3×9Ó9Ù%/°Ñ%^ÈVÐ%^ÐW]Ñ%^˜
Øà!Ð!ñ (Fð
 "Ð!r8   r!   )Údispatch_bnb_8bit)Údispatch_bnb_4bitrl   zTarget module zî is not supported. Currently, only the following modules are supported: `torch.nn.Linear`, `torch.nn.Embedding`, `torch.nn.Conv1d`, `torch.nn.Conv2d`, `torch.nn.Conv3d`, `transformers.pytorch_utils.Conv1D`, `torch.nn.MultiheadAttention.`.)rÏ   r—   r   ÚbnbrÔ   r   rÕ   Úextendr%   r"   r#   r&   r'   r(   r.   r/   r-   r,   ry   )
rœ   rV   r3   r5   ÚdispatchersrÒ   rÔ   rÕ   r­   Ú
dispatchers
             r6   r�   ÚLoraModel._create_new_modulel  sð   € ð ˆà×&×&ò"ð ×ÑÐ4Ô5ô ×ÑÝ.à×ÑÐ0Ô1ä ×"Ñ"Ý.à×ÑÐ0Ô1à×ÑäÜÜÜÜÜÜ Ü!Ü+Ü ðô	
ð ˆ
Û%ˆJÙ# FÑWÀÐWÐPVÑWˆJØÓ%Ùñ &ð
 ÑäØ   ð )Wð Wóð ð Ðr8   c           	   /  ó¢  ^#   • TR                  SS 5      nTR                  SS 5      nUc  Uc  S v •  g / nUGbz  [        R                  R                  [        R
                  5      [        R                  R                  S5      :  nU(       a  [        S5      eSSKJn  U R                  5        GH   u  p‰[        X—5      (       a¤  U	R                  (       a“  S n
S n[        U	S	/ 5      (       a  [        S
5      e/ U	l        U	R                  [        X¨US95      nU	R                  R!                  U5        U	R#                  [        X¸5      5      nU	R                  R!                  U5        [        U	[$        5      (       d  MÑ  [        [&        US9nU	R                  USS9nUR!                  U5        GM     TR)                  SS 5      n[        U[*        5      =(       a    US:„  nU(       a  Ub  [        S5      eUGb2  U R,                  (       a  [        S5      e[/        5       nU R1                  5        HT  n	[        U	[$        5      (       d  M  UU	R2                  R5                  5       -  nUU	R6                  R5                  5       -  nMV     U Vs1 s H  nUS:w  d  M  UiM     nnUU-
  nU(       a&  [        SSR9                  [;        U5      5       35      eUS S  nU(       aI  [        U[<        [>        45      (       d  [A        S[C        U5       S35      e[E        U4S jU 5       / 5      nU R1                  5        H^  n[        U[$        5      (       d  [        U[F        5      (       d  M/  [        [H        US9nUR                  USS9nUR!                  U5        M`     [K        U RL                  5      nU(       au  Ubr  UR1                  5        H^  n[        U[$        5      (       d  [        U[F        5      (       d  M/  [        [H        US9nUR                  USS9nUR!                  U5        M`     S v •  U H  nURO                  5         M     g s  snf 7f)Nr1   r:   z4.52.1z,Using aLoRA requires transformers >= 4.52.1.r   )ÚGradientCheckpointingLayerc                óÔ   • UR                  5        HT  n[        U[        5      (       d  M  UR                  [	        [
        US   S9SS9nUR                  R                  U5        MV     g )Nr:   ©r:   T©Úwith_kwargs)ÚmodulesrŠ   r*   Úregister_forward_pre_hookr   r;   Ú*_peft_gradient_checkpointing_forward_hooksr—   )rÊ   rË   Úinputsr5   Ú	submoduleÚhandles         r6   Úforward_pre_hookÚ>LoraModel._enable_peft_forward_hooks.<locals>.forward_pre_hookÒ  sb   € Ø)/¯©Ö)9˜IÜ)¨)´Y×?Ó?Ø)2×)LÑ)LÜ$+Ô,KÐ[aÐbqÑ[rÑ$sØ04ð *Mð *" ð !'× QÑ Q× XÑ XÐY_Ö `ò *:r8   c                óž   • UR                   (       a<  UR                   R                  5       R                  5         UR                   (       a  M;  g g rQ   )rã   ÚpopÚremove)rÊ   rË   Úgrad_outputr5   s       r6   Úbackward_hookÚ;LoraModel._enable_peft_forward_hooks.<locals>.backward_hookÛ  s4   € Ø$×O×OØ"×MÑM×QÑQÓS×ZÑZÔ\ð %×O×OÓOr8   rã   z©Multiple invocations of PEFT forward hooks before .backward() with enabled gradient checkpointing. Disable gradient checkpointing or only call forward once per backward.rÞ   Trß   Ú	num_beamsr!   z(Beam search not yet supported for aLoRA.z?Cannot pass `adapter_names` when the model is in training mode.Ú__base__z.Trying to infer with non-existing adapter(s): z, zGot adapter names of type z, expected a list of str.c              3  ó4   >#   • U  H  o/TS    -  v •  M     g7f)rï   Nr2   )rS   Únr5   s     €r6   rW   Ú7LoraModel._enable_peft_forward_hooks.<locals>.<genexpr>  s   øé € Ð$VÊÀ1 S¨6°+Ñ+>Ö%>Êùs   ƒ)r1   )(rê   Ú	packagingÚversionÚparseÚtransformersÚ__version__ry   Útransformers.modeling_layersrÜ   rÄ   rŠ   Úgradient_checkpointingr€   rã   râ   r   r—   Úregister_full_backward_hookr*   r;   r   ÚintÚtrainingÚsetrá   rZ   r}   r‚   ÚjoinÚsortedÚlistÚtupleÚ	TypeErrorÚtypeÚsumr   r7   rA   r?   rë   )rJ   r4   r5   r1   r:   Úhook_handlesÚtransformers_lt_4_52rÜ   rò   Úlayerrç   rí   ræ   Úpre_forwardrï   Úuses_beam_searchÚexpected_adaptersrÊ   Úunique_adaptersÚunexpected_adaptersÚoriginal_adapter_namesrË   r@   s     `                    r6   Ú_enable_peft_forward_hooksÚ$LoraModel._enable_peft_forward_hooks°  sÛ  øé € ð Ÿ
™
 ?°DÓ9ˆØŸ
™
 ?°DÓ9ˆàÑ  ]Ñ%:ãØØˆàÒ$ô $-×#4Ñ#4×#:Ñ#:¼<×;SÑ;SÓ#TÔW`×WhÑWh×WnÑWnØóXñ $Ð ö $Ü Ð!OÓPÐPåOà ×.Ñ.×0‘�ô ˜e×@Ñ@ÀU×Ea×Eaòaò]ô ˜uÐ&RÐTV×WÑWÜ(ðtóð ð HJ�EÔDØ"×<Ñ<¼WÐEUÐhuÑ=vÓw�FØ×DÑD×KÑKÈFÔSØ"×>Ñ>¼wÀ}Ó?XÓY�FØ×DÑD×KÑKÈFÔSÜ˜e¤Y×/Ó/Ü")Ô*IÐYfÑ"g�KØ"×<Ñ<¸[ÐVZÐ<Ð[�FØ ×'Ñ'¨×/ñK 1ðL —J‘J˜{¨DÓ1ˆ	Ü% i´Ó5×I¸9Àq¹=ÐÞØÑ(Ü Ð!KÓLÐLØÒ$Ø�}�}Ü Ð!bÓcÐcô
 !$£ÐØŸ™ž�Ü˜e¤Y×/Ó/Ø%¨¯©×):Ñ):Ó)<Ñ<Ð%Ø%¨×)?Ñ)?×)DÑ)DÓ)FÑFÒ%ñ (ñ 1>ÓT²¨ÀÈÑASŸt±ˆOÐTØ"1Ð4EÑ"EÐÞ"Ü ØDÀTÇYÁYÌvÐViÓOjÓEkÐDlÐmóð ð
 &3±1Ð%5Ð"ÞÜ! -´$¼°×?Ñ?Ü#Ð&@ÄÀmÓATÐ@UÐUnÐ$oÓpÐpô !$Ô$VÉÓ$VÐXZÓ [�àŸ,™,ž.�Ü˜f¤i×0Ñ0´J¸vÔG_×4`Ó4`Ü")Ô*IÐYfÑ"g�KØ#×=Ñ=¸kÐW[Ð=Ð\�FØ ×'Ñ'¨Ö/ñ	 )ô # 4§:¡:Ó.ˆGÞ WÑ%8ð &Ÿo™oÖ/�FÜ! &¬)×4Ñ4¼
À6ÔKc×8dÓ8dô '.Ô.MÐ]sÑ&t˜Ø!'×!AÑ!AÀ+Ð[_Ð!AÐ!`˜Ø$×+Ñ+¨FÖ3ñ 0ó 	ã"ˆFØ�M‰MŽOò #ùòK Uùs5   ƒE8QÅ?CQÉ
AQÊ
Q
ÊQ
Ê CQÍ(BQÏ9AQc                óÄ   >• [         TU ]  5         [        U R                  SS5      S:X  a  [	        S5      eU R
                  R                  S5      (       a  [	        S5      eg)z{Verify that the configuration supports merging.

Currently gptq quantization and replicated layers do not support merging.
Úquantization_methodNrf   z9Cannot merge LORA layers when the model is gptq quantizedrI   z>Cannot merge LORA layers when base model layers are replicated)ÚsuperÚ_check_merge_allowedr€   r?   ry   rK   r   )rJ   Ú	__class__s    €r6   r  ÚLoraModel._check_merge_allowed&  s[   ø€ ô
 	‰Ñ$Ô&Ü�4—:‘:Ð4°dÓ;¸vÓEÜÐXÓYÐYØ×Ñ×ÑÐ 3×4Ñ4ÜÐ]Ó^Ð^ð 5r8   c                óú   • UR                   cm  U R                  R                  US   5      nUb0  [        U[        5      (       a  X1l         U$ [        U5      Ul          U$ UR                  (       d  [        S5      eU$ )NÚ
model_typezFPlease specify `target_modules` or `target_parameters`in `peft_config`)Útarget_modulesÚtarget_module_mappingr   rŠ   rF   rþ   rR   ry   )rJ   rK   Úmodel_configr  s       r6   Ú_prepare_adapter_configÚ!LoraModel._prepare_adapter_config1  s~   € Ø×%Ñ%Ñ-Ø!×7Ñ7×;Ñ;¸LÈÑ<VÓWˆNØÑ)Ü˜n¬c×2Ñ2Ø1?Ô.ð
 Ðô 25°^Ó1D�KÕ.ð Ðð !×2×2Ü Ð!iÓjÐjØÐr8   c                ó¶  ^ ^• U H9  nU[        T R                  R                  5       5      ;  d  M,  [        SU S35      e   U H0  nT R                  U   R                  (       d  M#  [        SU S35      e   T R                  5        Vs/ s H  n[        U[        5      (       d  M  UPM     nnU V^s/ s H   m[        U4S jU 5       5      S:”  d  M  TPM"     nnU(       a  [        S[        U5       S35      e[        U5      S:X  a  S	OUnU 4S
 jU 5        V	s/ s HN  n	U	R                  (       d  U	R                  O-[        U	R                  /U	R                  R                  5       Q76 PMP     n
n	US;   a)  [        [        U
5      5      S:w  a  [        S5      eU
S   nOKUS:X  a  [        U
5      nO9UR                  S5      (       a  U=(       d    [        U
5      nO[        SU 35      eU Vs/ s H%  n[!        T R                  U   R"                  5      PM'     nnU(       d  [        SU 35      e[        [        U5      5      S:”  a  [        S5      eUS   [$        L a  SR'                  U 4S jU 5       5      nOCUS   [        L a%  [)        [*        R,                  U 4S jU 5       5      nO[/        SUS    S35      eX+U4$ s  snf s  snf s  sn	f s  snf )zw
Helper function to check if the arguments to add_weighted_adapter are valid and compatible with the underlying
model.
zAdapter z does not existzSadd_weighted_adapter does not support targeting nn.Parameter (problematic adapter 'z')c              3  ó@   >#   • U  H  oTR                   ;   v •  M     g 7frQ   )Úmodules_to_save)rS   ÚadapterÚwrappers     €r6   rW   Ú8LoraModel._check_add_weighted_adapter.<locals>.<genexpr>U  s   øé € ÐNÂX¸'˜g×5Ñ5Ö5ÂXùrÁ   r!   z\Cannot add weighted adapters if they target the same module with modules_to_save, but found z such instance(s).Úlinearc              3  óB   >#   • U  H  nTR                   U   v •  M     g 7frQ   )rK   ©rS   r!  rJ   s     €r6   rW   r#  c  s   øé € ÐMÂH¸˜4×+Ñ+¨GÖ4ÂHùs   ƒ)r$  r   r   r   r   zkAll adapters must have the same r value when using combination_type linear, ties, dare_ties or dare_linear.r   ÚcatÚsvdzInvalid combination_type: z'Found no adapter matching the names in z¢all adapter configs should follow the same target modules type. Combining adapters with `target_modules` type being a mix of list/set and string is not supported.Ú|c              3  ó^   >#   • U  H"  nS TR                   U   R                   S3v •  M$     g7f)Ú(Ú)N©rK   r  r&  s     €r6   rW   r#  ‚  s0   øé € Ð)rÒiqÐ^e¨A¨d×.>Ñ.>¸wÑ.G×.VÑ.VÐ-WÐWXÕ*YÒiqùs   ƒ*-c              3  óV   >#   • U  H  nTR                   U   R                  v •  M      g 7frQ   r-  r&  s     €r6   rW   r#  …  s$   øé € Ð`ÒW_ÈG˜t×/Ñ/°Ñ8×GÖGÒW_ùs   ƒ&)zInvalid type z found in target_modules)r  rK   r}   ry   rR   rá   rŠ   r   r  Úlenr|   r^   ÚmaxÚvaluesrþ   Úendswithr  r  rF   rÿ   r   r†   Úor_r  )rJ   ÚadaptersÚcombination_typeÚsvd_rankr!  rË   Úmodules_to_save_wrappersr"  Úproblematic_wrappersrl   Úadapters_ranksÚnew_rankÚtarget_module_typesÚnew_target_moduless   `      `      r6   Ú_check_add_weighted_adapterÚ%LoraModel._check_add_weighted_adapter=  sñ  ù€ ó  ˆGØœd 4×#3Ñ#3×#8Ñ#8Ó#:Ó;Õ;Ü  8¨G¨9°OÐ!DÓEÐEñ  ó  ˆGØ×Ñ Ñ(×:×:Ñ:Ü ØiÐjqÐirÐrtÐuóð ñ  ð :>¿¹¼Ó#tº¨vÌ:ÐV\Ô^r×Ks§F¹Ð Ð#tñ 4ô 
â3�ÜÔNÁXÓNÓNÐQRÑR÷ Ù3ð 	ð  
ö
  ÜØnÜÐ+Ó,Ð-Ð-?ðAóð ô (+¨8£}¸Ó'9™8Ð?OÐô
 NÁHÔMó%
ò N�ð #×/×/ˆF�HŠH´S¸¿¹Ð5aÀF×DWÑDW×D^ÑD^ÓD`Ò5aÒaÙMð 	ð %
ð Ð`Ó`ä”3�~Ó&Ó'¨1Ó,Ü ð#óð ð & aÑ(‰HØ Ó&ô ˜>Ó*‰HØ×&Ñ& u×-Ñ-à×6¤3 ~Ó#6‰HäÐ9Ð:JÐ9KÐLÓMÐMá]eÓfÒ]eÐRYœt D×$4Ñ$4°WÑ$=×$LÑ$LÖMÑ]eÐÐfÞ"ÜÐFÀxÀjÐQÓRÐRÜŒsÐ&Ó'Ó(¨1Ó,Üðuóð ð
 ˜qÑ!¤SÒ(Ø!$§¡Ô)rÑiqÓ)rÓ!rÑØ  Ñ#¤sÒ*Ü!'Ü—‘Ô`ÑW_Ó`ó"Ñô ˜mÐ,?ÀÑ,BÐ+CÐC[Ð\Ó]Ð]àÐ+=Ð=Ð=ùòs $uùò 
ùò%
ùò0 gs%   Â
KÂ'KÂ4KÃKÄAKÇ2,Kc                óö  • U[        U R                  R                  5       5      ;   a  gU R                  UUUS9u  pKn[	        U R                  US      UUU0 0 S9U R                  U'   U R                  U R                  U5        [        U R                  U5        U R                  R                  5        VVs/ s H  u  pÞU R                  U;  d  M  UPM     nnnU GH“  n[        U R                  U5      u  nnn[        U[        5      (       d  M5  UUR                  ;   a3  UR                  U   R                  nUR                  U   R                  nO1UUR                   ;   a  UR                   U   nUR"                  U   nOM©  UR$                  S-  Ul        UR$                  S-  Ul        US:X  Gaa  / / nn['        X5       HÃ  u  nnUUR                  ;   a3  UR                  U   R                  nUR                  U   R                  nO1UUR                   ;   a  UR                   U   nUR"                  U   nOMz  UR)                  UR$                  U-  UR*                  U   -  5        UR)                  UR$                  5        MÅ     [-        U5      S:X  a  [/        S5      e[0        R2                  " USS9n[0        R2                  " US	S9nUUR$                  SUR4                  S   2SS24'   UUR$                  SS2SUR4                  S	   24'   GM9  US
;   a*  U R7                  UUUUUUUU	U
UUUS9u  Ul        Ul        GMi  US;   d  GMr  U R9                  XAUUXš5      u  Ul        Ul        GM–     gs  snnf )a(	  
This method adds a new adapter by merging the given adapters with the given weights.

When using the `cat` combination_type you should be aware that rank of the resulting adapter will be equal to
the sum of all adapters ranks. So it's possible that the mixed adapter may become too big and result in OOM
errors.

Args:
    adapters (`list`):
        List of adapter names to be merged.
    weights (`list`):
        List of weights for each adapter. Weights can be positive or negative, allowing for both addition and
        subtraction of adapter effects.
    adapter_name (`str`):
        Name of the new adapter.
    combination_type (`str`):
        The merging type can be one of [`svd`, `linear`, `cat`, `ties`, `ties_svd`, `dare_ties`, `dare_linear`,
        `dare_ties_svd`, `dare_linear_svd`, `magnitude_prune`, `magnitude_prune_svd`]. When using the `cat`
        combination_type, the rank of the resulting adapter is equal to the sum of all adapters ranks (the
        mixed adapter may be too big and result in OOM errors).
    svd_rank (`int`, *optional*):
        Rank of output adapter for svd. If None provided, will use max rank of merging adapters.
    svd_clamp (`float`, *optional*):
        A quantile threshold for clamping SVD decomposition output. If None is provided, do not perform
        clamping. Defaults to None.
    svd_full_matrices (`bool`, *optional*):
        Controls whether to compute the full or reduced SVD, and consequently, the shape of the returned
        tensors U and Vh. Defaults to True.
    svd_driver (`str`, *optional*):
        Name of the cuSOLVER method to be used. This keyword argument only works when merging on CUDA. Can be
        one of [None, `gesvd`, `gesvdj`, `gesvda`]. For more info please refer to `torch.linalg.svd`
        documentation. Defaults to None.
    density (`float`, *optional*):
        Value between 0 and 1. 0 means all values are pruned and 1 means no values are pruned. Should be used
        with [`ties`, `ties_svd`, `dare_ties`, `dare_linear`, `dare_ties_svd`, `dare_linear_svd`,
        `magnintude_prune`, `magnitude_prune_svd`]
    majority_sign_method (`str`):
        The method, should be one of ["total", "frequency"], to use to get the magnitude of the sign values.
        Should be used with [`ties`, `ties_svd`, `dare_ties`, `dare_ties_svd`]
N)r4  r5  r6  r   )r^   r_   r  r~   r|   g        r'  z9No matching LoRAs found. Please raise an issue on GitHub.©Údimr!   )r(  Úties_svdÚdare_linear_svdÚdare_ties_svdÚmagnitude_prune_svd©Úfull_matricesÚdriver)r$  r   r   r   r   )r  rK   r}   r=  r   Úinject_adapterr?   r   rÄ   rG   r   rŠ   r*   rZ   r»   r[   r‚   rƒ   Údatar™   r—   Úscalingr/  ry   rÃ   r'  ÚshapeÚ1_svd_generalized_task_arithmetic_weighted_adapterÚ-_generalized_task_arithmetic_weighted_adapter)rJ   r4  ÚweightsrV   r5  r6  Ú	svd_clampÚsvd_full_matricesÚ
svd_driverÚdensityÚmajority_sign_methodr:  r<  rT   Ú_Úkey_listr3   Útarget_lora_AÚtarget_lora_BÚloras_AÚloras_Br!  r»   Úcurrent_adapter_lora_AÚcurrent_adapter_lora_Bs                            r6   Úadd_weighted_adapterÚLoraModel.add_weighted_adapterŒ  sf  € ðl œ4 × 0Ñ 0× 5Ñ 5Ó 7Ó8Ó8Øà9=×9YÑ9YØØ-Øð :Zð :
Ñ6ÐÐ$6ô *1Ø×Ñ˜X a™[Ñ)ØØØ-ØØñ*
ˆ×Ñ˜Ñ&ð 	×Ñ˜DŸJ™J¨Ô5ô 	˜Ÿ
™
 LÔ1à&*§j¡j×&>Ñ&>Ô&@Ô[Ò&@™F˜CÀDÇKÁKÐWZÑDZ—CÑ&@ˆÑ[ÜˆCÜ*¨4¯:©:°sÓ;‰LˆAˆv�qÜ˜&¤)×,Ó,Ø 6§=¡=Ó0Ø$*§M¡M°,Ñ$?×$FÑ$F�MØ$*§M¡M°,Ñ$?×$FÑ$F‘MØ! V×%<Ñ%<Ó<Ø$*×$;Ñ$;¸LÑ$I�MØ$*×$;Ñ$;¸LÑ$I‘Máà%2×%7Ñ%7¸#Ñ%=�Ô"Ø%2×%7Ñ%7¸#Ñ%=�Ô"Ø# uÔ,Ø')¨2˜W�GÜ+.¨xÖ+A™˜ Ø" f§m¡mÓ3Ø5;·]±]À7Ñ5K×5RÑ5RÐ2Ø5;·]±]À7Ñ5K×5RÑ5RÑ2Ø$¨×(?Ñ(?Ó?Ø5;×5LÑ5LÈWÑ5UÐ2Ø5;×5LÑ5LÈWÑ5UÑ2á$ØŸ™Ð'=×'BÑ'BÀVÑ'KÈfÏnÉnÐ]dÑNeÑ'eÔfØŸ™Ð'=×'BÑ'BÖCñ ,Bô ˜7“| qÓ(Ü(Ð)dÓeÐeÜ#Ÿiši¨°QÑ7�GÜ#Ÿiši¨°QÑ7�GØ@G�M×&Ñ&Ð'9¨¯©°qÑ)9Ð'9º1Ð'<Ñ=Ø@G�M×&Ñ&¢qÐ*<¨G¯M©M¸!Ñ,<Ð*<Ð'<Ô=Ø%ð *ó ð >B×=sÑ=sØ(Ø ØØ ØØ%Ø%ØØ,Ø!Ø&7Ø)ð >tð >Ñ:�MÔ&¨×(:ð &Ð)jÖjØ=A×=oÑ=oØ(°G¸VÀWó>Ñ:�MÔ&¨×(:òq ùó \s   Â7M5ÃM5c                ó0  ^• / n/ n[        U4S jU 5       5      n[        X#5       HZ  u  nnUTR                  ;   d  UTR                  ;   d  M(  UR	                  U5        UR	                  UTR
                  U   -  5        M\     [        U5      S:X  a  [        S5      eU Vs/ s H  nTR                  U5      PM     nn[        R                  " U5      R                  US   R                  5      nUS:X  a  [        UU5      nO\US:X  a  [        UXèU	5      nOHUS:X  a  [        UXè5      nO5US:X  a  [!        UXèU	5      nO!US:X  a  [#        UXè5      nO[        S	U 35      e[%        T[&        5      nU(       aG  TR(                  R+                  5       S
S S:H  nU(       d  UR-                  SS9nOUR/                  5       n[1        TS5      (       a  TR2                  (       d  U(       a  UR4                  n[        R6                  R9                  UX¼S9u  nnnUS S 2S U24   nUS U nU[        R:                  " U5      -  nUS U2S S 24   nU
br  [        R<                  " UR-                  5       UR-                  5       /5      n[        R>                  " UU
5      nU* nURA                  UU5      nURA                  UU5      nU(       aJ  URC                  URD                  RF                  5      nURC                  URD                  RF                  5      nUU4$ s  snf )Nc              3  ó@   >#   • U  H  oTR                   ;   v •  M     g 7frQ   )r‚   )rS   r!  r3   s     €r6   rW   ÚNLoraModel._svd_generalized_task_arithmetic_weighted_adapter.<locals>.<genexpr>&  s   øé € ÐVÊXÀ' f×&=Ñ&=Ö=ÊXùrÁ   r   z9No matching LoRAs found. Please raise an issue on Github.r(  rB  rC  rD  rE  z*Invalid value passed to combination type: é   é   )r!   r!   r!   )Ú	start_dimÚfan_in_fan_outrF  )$rz   r™   rZ   r‚   r—   rK  r/  ry   Úget_delta_weightrÃ   ÚtensorrÇ   r¾   r   r   r   r   r   rŠ   r)   r»   ÚsizeÚflattenÚsqueezer>   re  ÚTÚlinalgr(  Údiagr'  ÚquantileÚclampÚreshaperJ  rL  )rJ   r5  r4  rO  r:  r3   rW  rX  rS  rT  ro  rG  rH  Úvalid_adaptersÚvalid_weightsÚis_embeddingr!  r»   Údelta_weightÚconv2dÚ
conv2d_1x1ÚUÚSÚVhÚdistÚhi_valÚlow_vals        `                     r6   rM  Ú;LoraModel._svd_generalized_task_arithmetic_weighted_adapter  sÉ  ø€ ð ˆØˆÜÔVÉXÓVÓVˆÜ" 8Ö5‰OˆG�VØ˜&Ÿ-™-Ó'¨7°f×6MÑ6MÕ+MØ×%Ñ% gÔ.Ø×$Ñ$ V¨f¯n©n¸WÑ.EÑ%EÖFñ  6ô ˆ~Ó !Ó#ÜÐXÓYÐYÙHVÓWÊ¸W˜×/Ñ/°Ö8ÉˆÐWÜŸš ]Ó3×6Ñ6°|ÀA±×7MÑ7MÓNˆØ˜uÓ$Ü*¨<¸ÓG‰LØ Ó+Ü ¨mÐFZÓ[‰LØÐ!2Ó2Ü& |°]ÓL‰LØ Ó0Ü$ \°=ÐK_Ó`‰LØÐ!6Ó6Ü*¨<¸ÓP‰LäÐIÐJZÐI[Ð\Ó]Ð]ä˜F¤FÓ+ˆÞØŸ™×+Ñ+Ó-¨a°Ð2°fÑ<ˆJÞØ+×3Ñ3¸aÐ3Ð@‘à+×3Ñ3Ó5�Ü�FÐ,×-Ñ-°&×2G×2GÎLØ'Ÿ>™>ˆLô —<‘<×#Ñ# LÀÐ#Ð]‰ˆˆ1ˆbØŠa��(�ˆl‰OˆØˆiˆxˆLˆØ”—
’
˜1“ÑˆØ�	��	š1�ÑˆØÑÜ—9’9˜aŸi™i›k¨2¯:©:«<Ð8Ó9ˆDÜ—^’^ D¨%Ó0ˆFØ�gˆGØ—‘˜ Ó(ˆAØ—‘˜' 6Ó*ˆBÞØ—	‘	˜-×,Ñ,×2Ñ2Ó3ˆAØ—‘˜M×.Ñ.×4Ñ4Ó5ˆBØ�1ˆuˆùòQ Xs   Â"Lc                ó  • / n/ n/ n	/ n
[        X#5       GH(  u  p¼X´R                  ;   a3  UR                  U   R                  nUR                  U   R                  nO0X´R                  ;   a  UR                  U   nUR
                  U   nOMx  XÄR                  U   -  nUS:¼  a  SOSnUR                  [        R                  " [        U5      5      U-  5        UR                  [        R                  " [        U5      5      5        U	R                  UR                  5        U
R                  UR                  5        GM+     [        R                  " U5      R                  U	S   R                  5      n[        R                  " U5      R                  U
S   R                  5      nXx/nXš/nU	S   R                   n[#        U5       H•  u  nnUS:X  a  [%        UUU   5      UU'   M   US:X  a  ['        UUU   XV5      UU'   M;  US:X  a  [)        UUU   U5      UU'   MV  US:X  a  [+        UUU   XV5      UU'   Mq  US:X  a  [-        UUU   U5      UU'   MŒ  [/        S	5      e   U Vs/ s H  nUR                  U5      PM     nnU$ s  snf )
Nr   r!   éÿÿÿÿr$  r   r   r   r   zInvalid combination type)r™   rZ   r»   r[   r‚   rƒ   rK  r—   ÚmathÚsqrtÚabsrJ  rÃ   rg  rÇ   r¾   ÚdtypeÚ	enumerater   r   r   r   r   ry   )rJ   r5  r4  rO  r3   rS  rT  Úvalid_weights_AÚvalid_weights_BÚlora_A_deltasÚlora_B_deltasr!  r»   r[  r\  Úweight_with_scalingÚsignrr  Úlora_deltasrƒ  ÚiÚtask_tensorsÚdeltas                          r6   rN  Ú7LoraModel._generalized_task_arithmetic_weighted_adapterY  sf  € ð ˆØˆØˆØˆÜ" 8×5‰OˆGØŸ-™-Ó'Ø)/¯©°wÑ)?×)FÑ)FÐ&Ø)/¯©°wÑ)?×)FÑ)FÑ&Ø×3Ñ3Ó3Ø)/×)@Ñ)@ÀÑ)IÐ&Ø)/×)@Ñ)@ÀÑ)IÑ&áà"(¯>©>¸'Ñ+BÑ"BÐØ+¨qÓ0‘1°bˆDà×"Ñ"¤4§9¢9¬SÐ1DÓ-EÓ#FÈÑ#MÔNØ×"Ñ"¤4§9¢9¬SÐ1DÓ-EÓ#FÔGØ× Ñ Ð!7×!<Ñ!<Ô=Ø× Ñ Ð!7×!<Ñ!<×=ñ!  6ô"  Ÿ,š, Ó7×:Ñ:¸=ÈÑ;K×;RÑ;RÓSˆÜŸ,š, Ó7×:Ñ:¸=ÈÑ;K×;RÑ;RÓSˆØ(Ð:ˆØ$Ð4ˆØ˜aÑ ×&Ñ&ˆÜ(¨Ö5‰OˆAˆ|Ø 8Ó+Ü!0°¸}ÈQÑ?OÓ!P�˜A“Ø! VÓ+Ü!% l°MÀ!Ñ4DÀgÓ!d�˜A“Ø! ]Ó2Ü!,¨\¸=ÈÑ;KÈWÓ!U�˜A“Ø! [Ó0Ü!*¨<¸ÀqÑ9IÈ7Ó!i�˜A“Ø!Ð%6Ó6Ü!0°¸}ÈQÑ?OÐQXÓ!Y�˜A“ä Ð!;Ó<Ð<ñ  6ñ 5@Ó@²K¨5�u—x‘x –±KˆÐ@ØÐùò As   É"Jc           
     ó*  • U R                   R                  5        H±  u  pEUR                  R                  [        R
                  :w  d  M/  UR                  R                  [        R                  :w  d  MY  UR                  R                  [        R                  :w  d  Mƒ  UR                  S5      (       d  M›  [        R                  " S5        M³     [        U UR                  SS5      US9n0 nUR                  5        H’  nSU;   aA  [        R                  " X   USR                  UR!                  S5      SS 5         /S	S
9Xt'   MJ  SU;   d  MR  [        R                  " X   USR                  UR!                  S5      SS 5         * /SS
9Xt'   M”     U$ )a  
This function can calculate the updates of the PiSSA/CorDA/OLoRA by comparing the parameters of the
PiSSA/CorDA/OLoRA adapter in `output_state_dict` with the initial values of PiSSA/CorDA/OLoRA in
`adapter_name`, thus converting PiSSA/CorDA/OLoRA to LoRA.
Úpissaa   Note that Quant(W_res) + AB != Quant(W) + \Delta(AB); the converted LoRA, when combined with W or Quant(W), may introduce a certain gap in the fine-tuned model. Therefore, we recommend directly using the Quant(W_res) in conjunction with the PiSSA adapter. Ú
state_dictN)r’  rV   rZ   Ú.r!   r   r@  r[   )r?   Únamed_parametersrJ  rƒ  rÃ   Úfloat32Úfloat16Úbfloat16Ú
startswithr“   r”   r   r   r}   r'  rÿ   Úsplit)rJ   Úoutput_state_dictrV   r5   rÊ   ÚparamÚmutated_init_state_dictÚtensors_loras           r6   Úsubtract_mutated_initÚLoraModel.subtract_mutated_init�  s`  € ð  Ÿ:™:×6Ñ6Ö8‰KˆDà—
‘
× Ñ ¤E§M¡MÕ1Ø—J‘J×$Ñ$¬¯©Õ5Ø—J‘J×$Ñ$¬¯©Õ6Ø×)Ñ)¨'×2Ó2Ü—’ðvöñ 9ô #<ØØ—z‘z ,°Ó5Ø%ñ#
Ðð
 ˆØ%×*Ñ*Ö,ˆDð ˜4ÓÜ%*§Y¢YØ&Ñ,Ð.EÀcÇhÁhÈtÏzÉzÐZ]ËÐ_`Ð_aÐObÓFcÑ.dÐeÐklñ&�Ó"ð ˜TÕ!Ü%*§Y¢YØ&Ñ,Ð/FÀsÇxÁxÐPT×PZÑPZÐ[^ÓP_Ð`aÐ`bÐPcÓGdÑ/eÐ.eÐfÐlmñ&�Ó"ñ -ð Ðr8   c                ór  • [        U5      nX!l        [        US/ 5      =(       d    / n[        U R                  U R                  R                  5       5      nXC;  a  UR                  U5        U H@  nU H7  n[        R                  " SU S3U5      (       d  M%  UR                  U5          M>     MB     X1l
        g)aW  
Add embedding layer to `modules_to_save` and remove rest of the tied layers from `module_to_save`. Maintain a
separate set for layers to be tied in `peft_config.tied_weights_keys`.

Args:
    peft_config (LoraConfig) -- The configuration of the Lora model.
    tied_weight_keys (list[str]) -- Contains the layers tied to the embedding layer.
r   ú(^|.*\.)ú($|\..*)N)rþ   Úmodules_to_tier€   r   r?   r�   r—   r•   Úmatchrë   r   )rJ   rK   Útied_weight_keysr   Úembed_layer_namerT   Úms          r6   Ú_add_modules_to_save_to_tieÚ%LoraModel._add_modules_to_save_to_tie³  s¨   € ô Ð/Ó0ÐØ%5Ô"ä! +Ð/@À"ÓE×KÈˆÜ8¸¿¹ÀTÇZÁZ×EdÑEdÓEfÓgÐð Ó2Ø×"Ñ"Ð#3Ô4ó $ˆCÛ$�Ü—8’8˜x¨ s¨(Ð3°S×9Ó9Ø#×*Ñ*¨1Ô-Úó %ñ $ð '6Õ#r8   c                óð  • [        U5      nX!l        [        USS5      n[        U R                  U R                  R                  5       5      n[        U[        5      (       a$  SU S[        R                  " U5       S3nX1l
        g[        U=(       d    / 5      nUR                  U5        U H@  nU H7  n[        R                  " SU S3U5      (       d  M%  UR                  U5          M>     MB     XQl
        g)aY  
Add embedding layer to `target_modules` and remove rest of the tied layers from `target_modules`. Maintain a
separate set for layers to be tied in `peft_config.target_modules_to_tie`

Args:
    peft_config (LoraConfig) -- The configuration of the Lora model.
    tied_weight_keys (list[str]) -- Contains the layers tied to the embedding layer.
r  Nz(?:z|^z$)r¡  r¢  )rþ   rY   r€   r   r?   r�   rŠ   rF   r•   Úescaper  Úaddr¤  rë   )rJ   rK   r¥  Úraw_target_modulesr¦  r  rT   r§  s           r6   Ú_add_targets_to_tieÚLoraModel._add_targets_to_tieÖ  sè   € ô Ð/Ó0ÐØ,<Ô)ä$ [Ð2BÀDÓIÐä8¸¿¹ÀTÇZÁZ×EdÑEdÓEfÓgÐäÐ(¬#×.Ñ.ð %(Ð(:Ð';¸2¼b¿iºiÐHXÓ>YÐ=ZÐZ\Ð!]ÐØ);Ô&ØäÐ/×5°2Ó6ˆØ×ÑÐ+Ô,ó $ˆCÛ#�Ü—8’8˜x¨ s¨(Ð3°S×9Ó9Ø"×)Ñ)¨!Ô,Úó $ñ $ð &4Õ"r8   r2   )rK   r$   r?   ú	nn.Module)rN   zOptional[str]ÚreturnÚNone)r4  ú	list[str]r5  rF   r6  ú
int | Noner±  ztuple[str, int, str])r(  NNTNNÚtotal)r4  r³  rO  zlist[float]rV   rF   r5  rF   r6  r´  rP  r´  rQ  ÚboolrR  z
str | NonerS  zfloat | NonerT  zLiteral['total', 'frequency']r±  r²  )NTNrQ   )rš  zdict[str, torch.Tensor]rV   rF   )rK   r$   r¥  r³  )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rG   Ú__annotations__r*   Útuner_layer_clsr   r  rL   r´   r�   Ústaticmethodr�   r   r  r  r  r=  r]  rM  rN  rž  r¨  r®  Ú__static_attributes__Ú__classcell__)r  s   @r6   rC   rC   X   su  ø‡ ñFðP €FˆCÓØ€OØNÐôCð, )-ñ\ð &ð\ð 
õ\ò|-ð8 ñAó ðAðF ñsó ðsõj	_ò
ðM>Ø!ðM>Ø58ðM>ØDNðM>à	ôM>ðh !&Ø#Ø $Ø"&Ø!%Ø $Ø>EðGàðGð ðGð ð	Gð
 ðGð ðGð ðGð  ðGð ðGð ðGð <ðGð 
õGðh ØØôBòH2öh$ôL!6÷F)4ò )4r8   rC   )r?   r°  r±  znn.Module | None)NÚ
__future__r   r€  r†   r•   r“   Ú
contextlibr   Údataclassesr   Ú	functoolsr   r   Útypingr   r	   Úpackaging.versionrô   rÃ   r÷   r
   Úpeft.import_utilsr   r   r   Úpeft.tuners.tuners_utilsr   r   r   r   r   Ú
peft.utilsr   r   r   r   r   r   r   Úpeft.utils.integrationsr   Úpeft.utils.merge_utilsr   r   r   r   r   Úpeft.utils.otherr    rg   r"   rh   r#   rl   r$   Úeetqr%   rf   r&   Úhqqr'   Úincr(   r  r)   r*   r+   r,   Úter-   Útorchaor.   Útp_layerr/   r7   r;   rA   rC   r2   r8   r6   Ú<module>rÓ     sœ   ðõ #ã Û Û 	Û Ý %Ý ß %ß $ã Û Û Ý ç `Ñ `÷õ ÷÷ ñ õ +ß aÕ aÝ ,å Ý Ý Ý Ý Ý Ý ß DÓ DÝ +Ý %Ý 'òòô

ôg4�	õ g4r8   