ó
    >:jö2  ã                  ó–   • S SK Jr  S SKrS SKrS SKJr  S SKJr  S SKJ	r	J
r
  S SKJr  SSKJr  SS	KJr  SS
KJrJrJr   " S S\	5      rg)é    )ÚannotationsN)ÚConv1D)Ú	BaseTunerÚBaseTunerLayer)Ú6TRANSFORMERS_MODELS_TO_TINYLORA_TARGET_MODULES_MAPPINGé   )Ú _maybe_include_all_linear_layersé   )ÚTinyLoraConfig)Ú	EmbeddingÚLinearÚTinyLoraLayerc                  óô   ^ • \ rS rSr% SrSrS\S'   \r\	r
SU 4S jjrSS jrSS jrSS	 jrSU 4S
 jjr            SS jr\          SS j5       rSSU 4S jjjrSU 4S jjrSU 4S jjrSrU =r$ )ÚTinyLoraModelé!   a  
Creates TinyLoRA model from a pretrained transformers model.

TinyLoRA is an extremely parameter-efficient fine-tuning method that uses SVD decomposition of frozen weights and
projects a tiny trainable vector through fixed random tensors. Based on the paper "Learning to Reason in 13
Parameters" (arXiv:2602.04118).

Args:
    model ([`~transformers.PreTrainedModel`]): The model to be adapted.
    config ([`TinyLoraConfig`]): The configuration of the TinyLoRA model.
    adapter_name (`str`): The name of the adapter, defaults to `"default"`.
    low_cpu_mem_usage (`bool`, *optional*, defaults to `False`):
        Create empty adapter weights on meta device. Useful to speed up the loading process.

Returns:
    `torch.nn.Module`: The TinyLoRA model.

Example:
    ```python
    >>> from transformers import AutoModelForCausalLM
    >>> from peft import TinyLoraConfig, get_peft_model

    >>> base_model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m")
    >>> config = TinyLoraConfig(r=2, u=64, target_modules=["q_proj", "v_proj"])
    >>> model = get_peft_model(base_model, config)
    ```

**Attributes**:
    - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
    - **peft_config** ([`TinyLoraConfig`]): The configuration of the TinyLoRA model.
Ú	tinylora_ÚstrÚprefixc                ó*   >• [         TU ]  " XX440 UD6  g )N)ÚsuperÚ__init__)ÚselfÚmodelÚconfigÚadapter_nameÚlow_cpu_mem_usageÚkwargsÚ	__class__s         €ÚW/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/tinylora/model.pyr   ÚTinyLoraModel.__init__F   s   ø€ Ü‰Ò˜¨ÑRÈ6ÓRó    c                óÖ   • X R                   ;   aZ  U R                   U   R                  5        H8  n[        R                  R	                  X1R
                  * UR
                  5        M:     gg)z@Re-initialize the tinylora_v vectors with uniform random values.N)Ú
tinylora_vÚvaluesÚnnÚinitÚuniform_Úinit_v_bound)r   r   r   Úvs       r   Ú_init_tinylora_vÚTinyLoraModel._init_tinylora_vI   sP   € àŸ?™?Ó*Ø—_‘_ \Ñ2×9Ñ9Ö;�Ü—‘× Ñ  ×%8Ñ%8Ð$8¸&×:MÑ:MÖNò <ð +r!   c                ó^   • [        U S5      (       d  [        R                  " 0 5      U l        gg)z>Initialize shared trainable vectors on first adapter creation.r#   N)Úhasattrr%   Ú
ModuleDictr#   )r   r   r   r   s       r   Ú_pre_injection_hookÚ!TinyLoraModel._pre_injection_hookO   s&   € ô �t˜\×*Ñ*Ü Ÿmšm¨BÓ/ˆD�Oð +r!   c                ó�  • U R                  U R                  5      nU R                  X5      n[        X0R                  5      n[        R
                  [        [        R                  [        4n0 nSnU R                  R                  5        H8  u  pxU R                  X75      (       d  M  [        X„5      (       d  M/  XeU'   US-  nM:     U$ )zêBuild an ordered mapping from target module key to index.

Iterates the model in the same order as ``inject_adapter`` to assign each target module a deterministic index
used for group assignment (weight_tying) and projection seeding.
r   r
   )Úget_model_configr   Ú_prepare_adapter_configr	   r%   r   r   r   r   Únamed_modulesÚ_check_target_module_existsÚ
isinstance)	r   r   Úmodel_configÚpeft_configÚtarget_typesÚmappingÚidxÚkeyÚmodules	            r   Ú_build_target_key_mappingÚ'TinyLoraModel._build_target_key_mappingU   s§   € ð ×,Ñ,¨T¯Z©ZÓ8ˆØ×2Ñ2°6ÓHˆÜ6°{ÇJÁJÓOˆô Ÿ	™	¤6¬2¯<©<¼ÐGˆà"$ˆØˆØŸ:™:×3Ñ3Ö5‰KˆCØ×3Ñ3°K×EÑEÙÜ˜&×/Ó/Ø"˜‘Ø�q‘’ñ 6ð ˆr!   c                óâ   >• [         TU ]  U5        [        U R                  R	                  5        Vs1 s H  o"R
                  iM     sn5      n[        U5      S:”  a  [        SU 35      egs  snf )z3Check the config when a new adapter is being added.r
   zgTinyLoRA projection tensors must be saved for all adapters or none, but got multiple different values: N)r   Ú_check_new_adapter_configÚsortedr8   r$   Úsave_projectionÚlenÚ
ValueError)r   r   ÚcÚsave_projection_unique_valuesr   s       €r   rA   Ú'TinyLoraModel._check_new_adapter_configm   ss   ø€ ä‰Ñ)¨&Ô1ä(.È4×K[ÑK[×KbÑKbÔKdÓ/eÒKdÀa×0AÔ0AÑKdÑ/eÓ(fÐ%ÜÐ,Ó-°Ó1ÜØyØ0Ð1ð3óð ð 2ùò 0fs   ²A,c                óÀ  • Uc  [        S5      e[        U S5      (       a  X`R                  ;  a  U R                  U5      U l        U R                  U   n[	        U R                  5      n	[        S[        U	SUR                  -
  -  5      5      n
[        SXš-  5      n[        X‹-  U
S-
  5      n[        U5      nX R                  ;  a#  [        R                  " 0 5      U R                  U'   XÐR                  U   ;  aâ  [        US5      (       a  UR                  R                  nOS n[        R                  " [         R"                  " UR$                  US95      nUR&                  SL a   [        R(                  R+                  U5        OEUR&                  S:X  a5  [        R(                  R-                  XñR.                  * UR.                  5        XðR                  U   U'   [1        U[2        5      (       a;  UR5                  U5        UR7                  UU R                  UUR8                  U5        OaU R;                  XR                  XÒU5      nUR5                  U5        X R<                  ;  a  UR?                  S	5        U RA                  XTUU5        X R<                  ;   aA  [C        US
S	5      nU(       d,  U R                  U   RE                  5        H
  nSUl#        M     g g g )NzCurrent Key shouldn't be `None`Ú_target_key_to_idxr
   g      ð?Úweight)ÚdtypeTÚuniformFÚinference_mode)$rE   r-   rJ   r>   rD   ÚmaxÚroundÚweight_tyingÚminr   r#   r%   ÚParameterDictrK   rL   Ú	ParameterÚtorchÚemptyÚuÚinit_weightsr&   Úzeros_r'   r(   r6   r   Úset_layer_idxÚupdate_layerÚrÚ_create_new_moduleÚactive_adapterÚrequires_grad_Ú_replace_moduleÚgetattrr$   Úrequires_grad)r   Útinylora_configr   ÚtargetÚtarget_nameÚparentÚcurrent_keyÚoptional_kwargsÚ	layer_idxÚnum_target_layersÚ
num_groupsÚ
group_sizeÚ	group_idxÚv_keyrL   r)   Ú
new_modulerN   Úparams                      r   Ú_create_and_replaceÚ!TinyLoraModel._create_and_replacex   st  € ð ÑÜÐ>Ó?Ð?ô �tÐ1×2Ñ2°k×I`ÑI`Ó6`Ø&*×&DÑ&DÀ_Ó&UˆDÔ#ð ×+Ñ+¨KÑ8ˆ	Ü × 7Ñ 7Ó8Ðô
 ˜œEÐ"3°s¸_×=YÑ=YÑ7YÑ"ZÓ[Ó\ˆ
Ü˜Ð-Ñ;Ó<ˆ
Ü˜	Ñ/°¸a±Ó@ˆ	Ü�I“ˆð Ÿ™Ó.Ü,.×,<Ò,<¸RÓ,@ˆD�O‰O˜LÑ)ð Ÿ™¨Ñ5Ó5ä�v˜x×(Ñ(ØŸ™×+Ñ+‘à�Ü—’œUŸ[š[¨×):Ñ):À%ÑHÓIˆAØ×+Ñ+¨tÒ3ä—‘—‘˜qÕ!Ø ×-Ñ-°Ó:Ü—‘× Ñ  ×%AÑ%AÐ$AÀ?×C_ÑC_Ô`à34�O‰O˜LÑ)¨%Ñ0ä�fœm×,Ñ,Ø× Ñ  Ô+Ø×ÑØØ—‘ØØ×!Ñ!Øõð ×0Ñ0°Ç/Á/ÐSXÐhnÓoˆJØ×$Ñ$ YÔ/à×#6Ñ#6Ó6à×)Ñ)¨%Ô0Ø× Ñ  °jÀ&ÔIð ×.Ñ.Ó.Ü$ _Ð6FÈÓNˆNÞ!Ø!Ÿ_™_¨\Ñ:×AÑAÖC�EØ*.�EÖ'ò Dð "ð /r!   c                ól  • [        U[        5      (       a  UR                  5       nOUn[        U[        R                  R
                  5      (       a@  U R                  (       a  [        R                  " S5        SU l        [        UUUUU 40 UD6nU$ [        U[        5      (       aE  SUS'   U R                  (       d  [        R                  " S5        SU l        [        UUUUU 40 UD6nU$ [        U[        R                  R                  5      (       a  [        UUUUU 40 UD6nU$ [        SU S35      e)Nzjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.FTÚis_target_conv_1d_layerzafan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.zTarget module z• is not supported. Currently, only the following modules are supported: `torch.nn.Linear`, `torch.nn.Embedding`, `transformers.pytorch_utils.Conv1D`.)r6   r   Úget_base_layerrU   r%   r   Úfan_in_fan_outÚwarningsÚwarnr   r   rE   )rc   r#   rn   r   rd   r   Útarget_base_layerro   s           r   r]   Ú TinyLoraModel._create_new_moduleÄ   s[  € ô �fœn×-Ñ-Ø &× 5Ñ 5Ó 7Ñà &ÐäÐ'¬¯©¯©×9Ñ9Ø×-×-Ü—’ð7ôð 27�Ô.ÜØØØØØñð ñˆJðL Ðô= Ð)¬6×2Ñ2Ø04ˆFÐ,Ñ-Ø"×1×1Ü—’Øwôð 26�Ô.ÜØØØØØñð ñˆJð. Ðô Ð)¬5¯8©8×+=Ñ+=×>Ñ>Ü"ØØØØØñð ñˆJð Ðô Ø   ð )`ð `óð r!   c                ó\  >• [         TU ]  X5        U(       d  g[        R                  [        R                  1nXR
                  ;   ae  U R
                  U   R                  5        HC  nUR                  U;   d  M  UR                  R                  [        R                  5      Ul        ME     gg)zp
Cast the adapter weights to the correct dtype.

Override to also handle the model-level tinylora_v parameters.
N)r   Ú_cast_adapter_dtyperU   Úfloat16Úbfloat16r#   r$   rL   ÚdataÚtoÚfloat32)r   r   Úautocast_adapter_dtypeÚdtypes_to_convert_to_fp32rp   r   s        €r   r|   Ú!TinyLoraModel._cast_adapter_dtype  s~   ø€ ô 	‰Ñ# LÔIæ%Øô &+§]¡]´E·N±NÐ$CÐ!ØŸ?™?Ó*ØŸ™¨Ñ6×=Ñ=Ö?�Ø—;‘;Ð";Õ;Ø!&§¡§¡¬u¯}©}Ó!=�E–Jò @ð +r!   c                ó^   >• [         TU ]  U5        XR                  ;   a  U R                  U	 gg)zCDelete an adapter and clean up the model-level shared v parameters.N)r   Údelete_adapterr#   )r   r   r   s     €r   r†   ÚTinyLoraModel.delete_adapter  s-   ø€ ä‰Ñ˜|Ô,ð Ÿ?™?Ó*Ø—‘ Ñ-ð +r!   c                ó¤  >• [         TU ]  U5        U R                  R                  5        H!  nUR                  5        H
  nSUl        M     M#     U R
                   Hq  nX@R                  ;   a#  [        U R                  U   SS5      nU(       a  M5  X@R                  ;   d  MF  U R                  U   R                  5        H
  nSUl        M     Ms     g)zÓ
Mark only the adapter layers as trainable.

Override the base class method to manage the shared tinylora_v parameters which are stored at the model level
and thus invisible to the base class's per-layer logic.
FrN   TN)r   Ú _mark_only_adapters_as_trainabler#   r$   rb   Úactive_adaptersr8   ra   )r   r   Úadapter_paramsrp   r^   rN   r   s         €r   r‰   Ú.TinyLoraModel._mark_only_adapters_as_trainable  s´   ø€ ô 	‰Ñ0°Ô7ð #Ÿo™o×4Ñ4Ö6ˆNØ'×.Ñ.Ö0�Ø&+�Ö#ó 1ñ 7ð #×2Ô2ˆNØ×!1Ñ!1Ó1Ü!(¨×)9Ñ)9¸.Ñ)IÐK[Ð]bÓ!c�Þ!ÙØ§¡Õ0Ø!Ÿ_™_¨^Ñ<×CÑCÖE�EØ*.�EÖ'ó Fò 3r!   )rJ   r#   )F)r   r   r   r   ÚreturnÚNone)r   ú	nn.Moduler   r   r   r   r�   rŽ   )r   r   r�   zdict[str, int])r   r   r�   rŽ   )rc   r   r   r   rd   r�   re   r   rf   r�   rg   r   )
rc   r   r#   znn.ModuleDictrn   r   r   r   rd   r�   )T)r   r   r‚   Úboolr�   rŽ   )r   r   r�   rŽ   )r   r�   r�   rŽ   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   Ú__annotations__r   Útuner_layer_clsr   Útarget_module_mappingr   r*   r/   r>   rA   rq   Ústaticmethodr]   r|   r†   r‰   Ú__static_attributes__Ú__classcell__)r   s   @r   r   r   !   sÞ   ø‡ ñð@ €FˆCÓØ#€OØRÐ÷SôOô0ô÷0	ðJ/à'ðJ/ð ðJ/ð ð	J/ð
 ðJ/ð ðJ/ð ôJ/ðX ð:Ø'ð:à!ð:ð ð:ð ð	:ð
 ó:ó ð:÷x>ñ >÷&.÷/õ /r!   r   )Ú
__future__r   rw   rU   Útorch.nnr%   Útransformers.pytorch_utilsr   Úpeft.tuners.tuners_utilsr   r   Ú
peft.utilsr   Útuners_utilsr	   r   r   Úlayerr   r   r   r   © r!   r   Ú<module>r¤      s:   ðõ #ã ã Ý Ý -ç >õõ <Ý "ß 3Ñ 3ôQ/�Iõ Q/r!   