ó
    >:jþ:  ã                   ó®   • S SK r S SKrS SKJr  S SKJrJr  S SKJrJ	r	  S SK
JrJr  S SKJrJrJrJrJr  S SKJr  SS	KJr  SS
KJrJrJr   " S S\	5      rg)é    N)ÚConv1D)Úis_bnb_4bit_availableÚis_bnb_available)Ú
LoraConfigÚ	LoraModel)ÚBaseTunerLayerÚget_device_map)Ú5TRANSFORMERS_MODELS_TO_ADALORA_TARGET_MODULES_MAPPINGÚ_freeze_adapterÚ_get_submodulesÚget_gptqmodel_quant_linearÚget_quantization_config)Úgather_params_ctxé   )ÚSVDQuantLinear)ÚAdaLoraLayerÚRankAllocatorÚ	SVDLinearc                   ó€   ^ • \ rS rSrSr\rU 4S jrS\SS4U 4S jjr	S r
\SS	 j5       rS
 rS rS rS rS rSrU =r$ )ÚAdaLoraModelé$   a”  
Creates AdaLoRA (Adaptive LoRA) model from a pretrained transformers model. Paper:
https://openreview.net/forum?id=lq62uWRJjiY

Args:
    model ([`transformers.PreTrainedModel`]): The model to be adapted.
    config ([`AdaLoraConfig`]): The configuration of the AdaLora model.
    adapter_name (`str`): The name of the adapter, defaults to `"default"`.
    low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
        Create empty adapter weights on meta device. Useful to speed up the loading process.

Returns:
    `torch.nn.Module`: The AdaLora model.

Example::

    >>> from transformers import AutoModelForSeq2SeqLM >>> from peft import LoraConfig, AdaLoraModel, AdaLoraConfig
    >>> config = AdaLoraConfig(
            peft_type="ADALORA", task_type="SEQ_2_SEQ_LM", init_r=12, lora_alpha=32, target_modules=["q", "v"],
            lora_dropout=0.01,
        )
    >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> model = AdaLoraModel(model, config, "default")

**Attributes**:
    - **model** ([`transformers.PreTrainedModel`]) -- The model to be adapted.
    - **peft_config** ([`AdaLoraConfig`]): The configuration of the AdaLora model.
c                 óž  >• [         TU ]  " XU40 UD6  SnU R                  R                  5        H  nUR                  (       a  M  US-  nM     US:”  a  [        S5      eU R                  U   R                  (       a  [        U R                  U5        g X0l        [        U R                  U R                  U   U R                  5      U l
        g )Nr   r   z›AdaLoraModel supports only 1 trainable adapter. When using multiple adapters, set inference_mode to True for all adapters except the one you want to train.)ÚsuperÚ__init__Úpeft_configÚvaluesÚinference_modeÚ
ValueErrorr   ÚmodelÚtrainable_adapter_namer   Úrankallocator)Úselfr   ÚconfigÚadapter_nameÚkwargsÚtraininable_mode_counterÚ	__class__s         €ÚV/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/adalora/model.pyr   ÚAdaLoraModel.__init__D   s»   ø€ Ü‰Ò˜¨Ñ?¸Ò?à#$Ð Ø×&Ñ&×-Ñ-Ö/ˆFØ×(×(Ñ(Ø(¨AÑ-Ò(ñ 0ð $ aÓ'Üð~óð ð
 ×Ñ˜LÑ)×8×8Ü˜DŸJ™J¨Õ5à*6Ô'Ü!.¨t¯z©z¸4×;KÑ;KÈLÑ;YÐ[_×[vÑ[vÓ!wˆDÕó    r#   ÚreturnNc                 óê   >• [         TU ]  U5        SnU R                  R                  5        H  nUR                  (       a  M  US-  nM     US:”  a"  [        U R                  R                   S35      eg)z´
A helper method to check the config when a new adapter is being added.

Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

r   r   z� supports only 1 trainable adapter. When using multiple adapters, set inference_mode to True for all adapters except the one you want to train.N)r   Ú_check_new_adapter_configr   r   r   r   r'   Ú__name__)r"   r#   r&   Úconfig_r'   s       €r(   r-   Ú&AdaLoraModel._check_new_adapter_configX   s~   ø€ ô 	‰Ñ)¨&Ô1à#$Ð Ø×'Ñ'×.Ñ.Ö0ˆGØ×)×)Ñ)Ø(¨AÑ-Ò(ñ 1ð $ aÓ'ÜØ—>‘>×*Ñ*Ð+ð ,%ð %óð ð (r*   c                 óR  • UR                   UR                  [        U R                  SS5      [        U R                  SS5      S.nUS   (       d
  US   (       a  [	        5       (       d  [        S5      e[        U R                  SS	9nUb  X‡S
'   [        U[        5      (       d^  [        U R                  5      n	U R                  " XU4SU	0UD6n
X R                  ;  a  U
R                  S5        U R                  XTX£5        g UR                  UUR                   UR                  US9  g )NÚis_loaded_in_8bitFÚis_loaded_in_4bit)ÚrÚ
lora_alphaÚloaded_in_8bitÚloaded_in_4bitr6   r7   z†To use AdaLora with 8-bit quantization, please install the `bitsandbytes` package. You can install it with `pip install bitsandbytes`.Úgptq)ÚmethodÚgptq_quantization_configÚ
device_map©r#   )Úinit_rr5   Úgetattrr   r   ÚImportErrorr   Ú
isinstancer   r	   Ú_create_new_moduleÚactive_adaptersÚrequires_grad_Ú_replace_moduleÚupdate_layer)r"   Úlora_configr$   ÚtargetÚtarget_nameÚparentÚcurrent_keyr%   Úquantization_configr;   Ú
new_modules              r(   Ú_create_and_replaceÚ AdaLoraModel._create_and_replacem   s   € ð ×#Ñ#Ø%×0Ñ0Ü% d§j¡jÐ2EÀuÓMÜ% d§j¡jÐ2EÀuÓMñ	
ˆð Ð#×$¨Ð/?×(@ÔJZ×J\ÑJ\ÜðFóð ô
 6°d·j±jÈÑPÐØÑ*Ø1DÐ-Ñ.ô ˜&¤,×/Ñ/Ü'¨¯
©
Ó3ˆJØ×0Ò0°ÈFÑtÐ_iÐtÐmsÑtˆJØ×#7Ñ#7Ó7à×)Ñ)¨%Ô0Ø× Ñ  °jÕIà×ÑØØ×"Ñ"Ø×&Ñ&Ø"ð	  ò r*   c                 ó&  • [        5       (       a
  SS KnSSKJn  [	        5       (       a  SSKJn  UR                  SS 5      n[        XƒS9n	UR                  SS5      n
UR                  S	S5      n[        U[        5      (       a  UR                  5       nOUnU
(       ay  [        UWR                  R                  5      (       aT  UR                  UR                  R                   UR                  R"                  UR$                  S
.5        W" X!4SU 0UD6nU$ U(       a˜  [	        5       (       a‰  [        UWR                  R&                  5      (       ad  UR)                  5       nUR                  UR*                  UR,                  R.                  UR,                  R0                  S.5        W" X!4SU 0UD6nU$ U	b   [        X)5      (       a  [3        X!4SU 0UD6nU$ [        U[4        R                  R6                  5      (       a/  U R8                  (       a  [:        R<                  " S5        SU l        OS[        U[>        5      (       a/  U R8                  (       d  [:        R<                  " S5        SU l        O[A        SU S35      e[C        X!4SU 0UD6nU$ )Nr   r   )ÚSVDLinear8bitLt)ÚSVDLinear4bitr:   )r;   r6   Fr7   )Úhas_fp16_weightsÚ	thresholdÚindexr#   )Úcompute_dtypeÚcompress_statisticsÚ
quant_typezjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.zafan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.TzTarget module zP is not supported. Currently, only `torch.nn.Linear` and `Conv1D` are supported.)"r   ÚbitsandbytesÚbnbrP   r   rQ   Úgetr   Úpopr@   r   Úget_base_layerÚnnÚLinear8bitLtÚupdateÚstaterR   rS   rT   Ú
Linear4bitÚcopyrU   ÚweightrV   rW   r   ÚtorchÚLinearÚfan_in_fan_outÚwarningsÚwarnr   r   r   )rF   r$   rG   r;   r%   rY   rP   rQ   r:   ÚQuantLinearr6   r7   Útarget_base_layerrL   Úfourbit_kwargss                  r(   rA   ÚAdaLoraModel._create_new_module–   sZ  € ô ×ÑÛ&å,Ü ×"Ñ"Ý*à#)§:¡:Ð.HÈ$Ó#OÐ ä0Ð1IÑaˆàŸ™Ð$4°eÓ<ˆØŸ™Ð$4°eÓ<ˆä�fœn×-Ñ-Ø &× 5Ñ 5Ó 7Ñà &ÐæœjÐ):¸C¿F¹F×<OÑ<O×PÑPØ�M‰Mà(9×(?Ñ(?×(PÑ(PØ!2×!8Ñ!8×!BÑ!BØ.×4Ñ4ñôñ )¨Ñ\ÀkÐ\ÐU[Ñ\ˆJðF ÐöE Ô 5× 7Ñ 7¼JÐGXÐZ]×Z`ÑZ`×ZkÑZk×<lÑ<lØ#Ÿ[™[›]ˆNØ×!Ñ!à%6×%DÑ%DØ+<×+CÑ+C×+WÑ+WØ"3×":Ñ":×"EÑ"Eñôñ ' vÑbÀKÐbÐSaÑbˆJð2 Ðð1 Ñ$¬°F×)HÑ)HÜ'¨Ñ[À[Ð[ÐTZÑ[ˆJð. Ðô+ Ð+¬U¯X©X¯_©_×=Ñ=Ø×-×-Ü—M’Mð;ôð 27�KÔ.øÜÐ-¬v×6Ñ6Ø"×1×1Ü—M’Mð:ôð 26�KÔ.øä Ø$ V Hð -Tð Uóð ô # 6ÑVÀÐVÈvÑVˆJàÐr*   c                 óÌ  • U R                   R                  " U0 UD6n[        USS 5      Gb§  [        UR                  [
        R                  5      (       Ga}  U R                  U R                     R                  nUS::  a  [        S5      eSnSnU R                   R                  5        GH  u  pxSU;   d  SU;   d  M  U R                  U;   d  M&  UR                  [
        R                  " S/5      :X  a6  [        X€S9   SU;   a  XˆR                  -  OUR                  U-  n	S S S 5        O#SU;   a  XˆR                  -  OUR                  U-  n	[
        R                   " W	R#                  5       S[
        R$                  " U	5      06n
SU
l        US	-  nU[
        R(                  " Xš-
  S
S9-  nGM     US:”  a  XV-  nOSnU=R                  XE-  -  sl        U$ ! , (       d  f       N—= f)NÚlossr   z*orth_reg_weight should be greater than 0. Úlora_AÚlora_B)Ú
fwd_moduleÚoutFr   Úfro)Úp)r   Úforwardr>   r@   rn   rd   ÚTensorr   r    Úorth_reg_weightr   Únamed_parametersÚshapeÚSizer   ÚTÚeyeÚsizeÚ
empty_likeÚrequires_gradÚnorm)r"   Úargsr%   Úoutputsrw   Ú	regu_lossÚ	num_paramÚnrt   Úpara_covÚIs              r(   ru   ÚAdaLoraModel.forwardÙ   s�  € Ø—*‘*×$Ò$ dÐ5¨fÑ5ˆä�G˜V TÓ*Ò6¼JÀwÇ|Á|ÔUZ×UaÑUa×<bÒ<bà"×.Ñ.¨t×/JÑ/JÑK×[Ñ[ˆOà !Ó#Ü Ð!MÓNÐNàˆIØˆIØŸ
™
×3Ñ3×5‘�Ø “M X°¥]¸×8SÑ8SÐWXÕ8XØ—w‘w¤%§*¢*¨a¨S£/Ó1Ü.¨qÓBØ2:¸a³- q¯3©3¢wÀQÇSÁSÈ1ÁW˜H÷ CÐBð /7¸!«m 1§s¡s¢7ÀÇÁÀqÁ˜ÜŸ	š	 8§=¡=£?ÐS¼×8HÒ8HÈÓ8RÑS�AØ&+�A”OØ ‘N�IØ¤§¢¨H©L¸EÑ!BÑB“Iñ 6ð ˜1‹}Ø%Ñ1‘	à�	Ø�LŠL˜OÑ7Ñ7�LØˆ÷ CÕBús   Ã<$GÇ
G#	c                 ó>  • U R                   U   nUR                  5        GHæ  u  pE[        U[        5      (       a  [	        U5      nOZ[        U[
        R                  5      (       a0  UR                  S5      nUR	                  5       R                  5       nO[        S5      eX$;   a#  SR                  UR                  S5      SS 5      O"SR                  UR                  S5      SS 5      n[        U R                  U5      u  p‰nU	R                  U   U   n
U	R                  U   U   nU	R                   U   S S 2U4   nU	R"                  U   nU	R%                  UUUR&                  US9  [
        R(                  " 5          US:”  ax  U	R                  U   R+                  U
5        U	R                  U   R+                  U5        U	R                   U   R+                  U5        U	R"                  U   R+                  U5        S S S 5        GMé     g ! , (       d  f       GMü  = f)NéÿÿÿÿzUnexpected type of rank_idxÚ.r   éþÿÿÿr<   )r   Úitemsr@   ÚlistÚsumrd   rv   ÚviewÚitemr   ÚjoinÚsplitr   r   Úlora_Ero   rp   ÚranknumrE   r5   Úno_gradÚcopy_)r"   Úrank_patternr$   rF   ÚnameÚrank_idxÚrankÚkeyÚ_rG   Úlora_E_weightsÚlora_A_weightsÚlora_B_weightsr•   s                 r(   Úresize_modules_by_rank_patternÚ+AdaLoraModel.resize_modules_by_rank_pattern÷   sÀ  € Ø×&Ñ& |Ñ4ˆØ*×0Ñ0×2‰NˆDÜ˜(¤D×)Ñ)Ü˜8“}‘Ü˜H¤e§l¡l×3Ñ3Ø#Ÿ=™=¨Ó,�Ø—|‘|“~×*Ñ*Ó,‘ä Ð!>Ó?Ð?Ø5AÓ5I�#—(‘(˜4Ÿ:™: c›?¨1¨RÐ0Ô1ÈsÏxÉxÐX\×XbÑXbÐcfÓXgÐhiÐjlÐXmÓOnˆCÜ*¨4¯:©:°sÓ;‰LˆA�qØ#Ÿ]™]¨<Ñ8¸ÑBˆNØ#Ÿ]™]¨<Ñ8¸ÑBˆNØ#Ÿ]™]¨<Ñ8º¸H¸ÑEˆNØ—n‘n \Ñ2ˆGØ×ÑØØØ×&Ñ&Ø"ð	  ñ ô —’•Ø˜!“8Ø—M‘M ,Ñ/×5Ñ5°nÔEØ—M‘M ,Ñ/×5Ñ5°nÔEØ—M‘M ,Ñ/×5Ñ5°nÔEà—N‘N <Ñ0×6Ñ6°wÔ?÷ !’ò) 3÷( !—ús   Å?A?HÈ
H	c                 ó¬  • UR                  5        H¿  u  pE[        U5      nX4;   a#  SR                  UR                  S5      SS 5      O"SR                  UR                  S5      SS 5      nS H^  nSU SU SU 3n	US:w  a$  XbU	   R                  S   :w  a  X)   U   OX)   X)'   M8  XbU	   R                  S   :w  a  X)   S S 2U4   OX)   X)'   M`     MÁ     U$ )	Nr‹   r   rŒ   rŠ   )r”   ro   rp   zbase_model.model.rp   r   )r�   r�   r’   r“   ry   )
r"   r˜   Ú
state_dictr$   r™   rš   r›   ÚprefixÚlayerrœ   s
             r(   Ú!resize_state_dict_by_rank_patternÚ.AdaLoraModel.resize_state_dict_by_rank_pattern  sö   € Ø*×0Ñ0Ö2‰NˆDÜ�x“=ˆDØ8DÓ8L�S—X‘X˜dŸj™j¨›o¨a°Ð3Ô4ÐRU×RZÑRZÐ[_×[eÑ[eÐfiÓ[jÐklÐmoÐ[pÓRqˆFÛ7�Ø)¨&¨°°5°'¸¸<¸.ÐI�Ø˜HÓ$à59È¹_×=RÑ=RÐSTÑ=UÓ5U˜
™¨Ò1Ð[eÑ[jð “Oð
 9=È3Á×@UÑ@UÐVWÑ@XÓ8X˜
™ª¨8¨Ò4Ð^hÑ^mð “Oó 8ñ 3ð Ðr*   c                 ó>  • U R                   U R                     nXR                  UR                  -
  :  a7  U R                  R                  U R                  U5      u  p4U(       a  XBl        ggXR                  UR                  -
  :X  aH  U R                  R                  U R                  USS9u  p4XBl        U R                  R                  5         gXR                  UR                  -
  :”  a1  U R                  R                  U R                  UR                  5        gg)aÝ  
This method updates Adalora budget and mask.

This should be called in every training step after `loss.backward()` and before `zero_grad()`.

`tinit`, `tfinal` and `deltaT` are handled with in the method.

Args:
    global_step (`int`): The current training step, it is used to calculate adalora budget.

Example:

```python
>>> loss = model(**input).loss
>>> loss.backward()
>>> optimizer.step()
>>> model.base_model.update_and_allocate(i_step)
>>> optimizer.zero_grad()
```
T)Ú
force_maskN)
r   r    Ú
total_stepÚtfinalr!   Úupdate_and_allocater   r˜   Ú	reset_iptÚmask_using_rank_pattern)r"   Úglobal_steprF   r�   r˜   s        r(   r­   Ú AdaLoraModel.update_and_allocate%  sô   € ð* ×&Ñ& t×'BÑ'BÑCˆà×/Ñ/°+×2DÑ2DÑDÓDØ"×0Ñ0×DÑDÀTÇZÁZÐQ\Ó]‰OˆAÞØ+7Õ(ð ð ×2Ñ2°[×5GÑ5GÑGÓGØ"×0Ñ0×DÑDÀTÇZÁZÐQ\ÐimÐDÐn‰OˆAð (4Ô$Ø×Ñ×(Ñ(Õ*ð ×1Ñ1°K×4FÑ4FÑFÓFØ×Ñ×6Ñ6°t·z±zÀ;×C[ÑC[Õ\ð r*   c                 óF   • [        U R                  R                   S35      e)z;This method is not supported for AdaLoRA, use LoRA instead.z. does not support add_weighted_adapter method.)Ú	TypeErrorr'   r.   )r"   r�   r%   s      r(   Úadd_weighted_adapterÚ!AdaLoraModel.add_weighted_adapterO  s    € ä˜4Ÿ>™>×2Ñ2Ð3Ð3aÐbÓcÐcr*   )r!   r    )N)r.   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r
   Útarget_module_mappingr   r   r-   rM   ÚstaticmethodrA   ru   r¡   r§   r­   r´   Ú__static_attributes__Ú__classcell__)r'   s   @r(   r   r   $   si   ø† ñð: RÐõxð(°
ð ¸t÷ ò*'ðR ó@ó ð@òDò<@ò<ò (÷Tdð dr*   r   )rg   rd   Útransformers.pytorch_utilsr   Úpeft.import_utilsr   r   Úpeft.tuners.lorar   r   Úpeft.tuners.tuners_utilsr   r	   Ú
peft.utilsr
   r   r   r   r   Úpeft.utils.integrationsr   r8   r   r¦   r   r   r   r   © r*   r(   Ú<module>rÅ      sA   ðó ã Ý -ç Eß 2ß C÷õ õ 6å  ß 9Ñ 9ômd�9õ mdr*   