ó
    >:j.  ã                   ó  • S SK Jr  S SKJr  S SKJrJr  S SKrS SKJ	s  J
r  S SKJ	r	  S SKJrJr  S SKJr  Sr S	\4S
 jr " S S\	R(                  5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      rg)é    )Údeepcopy©Úwraps)ÚAnyÚOptionalN)Únn)Údequantize_module_weightÚgather_params_ctx)Ú	transposeFÚ	cache_keyc                 ó   ^ • U 4S jnU$ )zÌCaching decorator for DoRA

Caching is only enabled if ENABLE_DORA_CACHING is set to True (default: False), when in eval mode, and when the
adapter_name is passed (e.g. not during layer initialization).

c                 ó4   >^ • [        T 5      UU 4S j5       nU$ )Nc                 ó  >• UR                  S5      n[        (       a  U R                  (       d  Uc  U R                  5         T" U /UQ70 UD6$ T SU 3nU R	                  US 5      nUb  U$ T" U /UQ70 UD6nU R                  XE5        U$ )NÚadapter_nameÚ-)ÚgetÚENABLE_DORA_CACHINGÚtrainingÚ_cache_clearÚ
_cache_getÚ_cache_store)ÚselfÚargsÚkwargsr   Úcache_key_adapterÚoutputr   Úfuncs         €€ÚR/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/lora/dora.pyÚwrapperÚ5cache_decorator.<locals>.cache_value.<locals>.wrapper(   sš   ø€ ð "Ÿ:™: nÓ5ˆLß'Ò'¨D¯M¯M¸lÑ>RØ×!Ñ!Ô#Ù˜DÐ2 4Ò2¨6Ñ2Ð2à#, +¨Q¨|¨nÐ =ÐØ—_‘_Ð%6¸Ó=ˆFØÑ!Ø�á˜$Ð0 Ò0¨Ñ0ˆFØ×ÑÐ/Ô8ØˆMó    r   )r   r   r   s   ` €r   Úcache_valueÚ$cache_decorator.<locals>.cache_value'   s    ù€ Ü	ˆt‹õ	ó 
ð	ð  ˆr!   © )r   r"   s   ` r   Úcache_decoratorr%      s   ø€ õð( Ðr!   c                   ó,  ^ • \ rS rSrU 4S jrS\S\SS4S jrS\S\\   S\\   4S	 jr	SS
 jr
SS\4U 4S jjjr\" S5      SS\\   S\R                  4S jj5       r\" S5      SS\\   4S jj5       rSS.SS jjrSSS.S jrS\4U 4S jjrSrU =r$ )ÚDoraLinearLayeré>   c                 ó<   >• [         TU ]  5         Xl        0 U l        g ©N)ÚsuperÚ__init__Úfan_in_fan_outÚ_dora_cache©r   r-   Ú	__class__s     €r   r,   ÚDoraLinearLayer.__init__?   s   ø€ Ü‰ÑÔØ,ÔØ+-ˆÕr!   ÚkeyÚvalueÚreturnNc                 ó    • X R                   U'   g r*   )r.   )r   r2   r3   s      r   r   ÚDoraLinearLayer._cache_storeD   s   € à %×Ñ˜Òr!   Údefaultc                 ó8   • U R                   R                  X5      $ r*   )r.   r   )r   r2   r7   s      r   r   ÚDoraLinearLayer._cache_getH   s   € à×Ñ×#Ñ# CÓ1Ð1r!   c                 ó8   • U R                   R                  5         g r*   )r.   Úclear)r   s    r   r   ÚDoraLinearLayer._cache_clearL   s   € Ø×Ñ×ÑÕ r!   Úmodec                 óP   >• U(       a  U R                  5         [        TU ]	  US9  U $ )N)r=   )r   r+   Útrain)r   r=   r0   s     €r   r?   ÚDoraLinearLayer.trainO   s%   ø€ ÞØ×ÑÔÜ‰‰˜4ˆÑ Øˆr!   úweight-normr   c                 ó¬   • [        XR                  5      nXU-  -   n[        R                  R	                  USS9R                  UR                  5      nU$ )Né   )Údim)r   r-   ÚtorchÚlinalgÚnormÚtoÚdtype)r   ÚweightÚlora_weightÚscalingr   Úweight_norms         r   Úget_weight_normÚDoraLinearLayer.get_weight_normU   sO   € ô ˜6×#6Ñ#6Ó7ˆØ KÑ/Ñ/ˆÜ—l‘l×'Ñ'¨°AÐ'Ð6×9Ñ9¸&¿,¹,ÓGˆØÐr!   úlora-weightc                 óà   • [         R                  " UR                  R                  S   UR                  R                  UR                  R
                  S9nU" U" U5      5      R                  nU$ )NrC   )ÚdevicerI   )rE   ÚeyerJ   ÚshaperR   rI   ÚT)r   Úlora_AÚlora_Br   Úx_eyerK   s         r   Úget_lora_weightÚDoraLinearLayer.get_lora_weight]   sV   € ô —	’	˜&Ÿ-™-×-Ñ-¨aÑ0¸¿¹×9MÑ9MÐU[×UbÑUb×UhÑUhÑiˆÙ™V E›]Ó+×-Ñ-ˆØÐr!   F)Úplace_on_cpuc                ó:  • UR                   [        R                  :H  nU(       a   UR                  5       nUR                  5       n[	        UR                  5       5         UR                  R                  S:X  a  [        U5      n[        U5      nUR                  R                  S:¼  ad  UR                  S   n[        R                  " UR                  SU/5      UR                  US/5      5      n	U	R                  UR                  5      n	OX2-  n	U(       a  U	R!                  5       n	U R#                  UR%                  UR&                  5      X”S9n
S S S 5        U(       a  W
R%                  S5      n
[(        R*                  " W
SS9U l        g ! , (       d  f       NA= f)	NÚ
Linear4bité   r   éÿÿÿÿ)rJ   rK   rL   ÚcpuT)Úrequires_grad)rI   rE   Úfloat16Úfloatr
   Ú
parametersr0   Ú__name__r   r	   ÚdataÚndimrT   ÚmmÚviewÚreshapeÚhalfrN   rH   rR   r   Ú	ParameterrJ   )r   Ú
base_layerrV   rW   rL   r[   Údtype_is_fp16rJ   ÚrrK   rM   s              r   Úupdate_layerÚDoraLinearLayer.update_layere   s7  € àŸ™¬¯©Ñ5ˆÞØ—\‘\“^ˆFØ—\‘\“^ˆFä˜z×4Ñ4Ó6Õ7Ø×#Ñ#×,Ñ,°Ó<ô & jÓ1�
ä-¨jÓ9ˆFØ�{‰{×Ñ 1Ó$Ø—L‘L ‘O�Ü#Ÿhšh v§{¡{°B¸°7Ó';¸V¿[¹[È!ÈRÈÓ=QÓR�Ø)×1Ñ1°&·,±,Ó?‘à$™o�æØ)×.Ñ.Ó0�Ø×.Ñ.Ø—y‘y §¡Ó/¸[ð /ð ˆK÷! 8ö( Ø%Ÿ.™.¨Ó/ˆKÜ—l’l ;¸dÑCˆ�÷- 8Õ7ús   ÁC3FÆ
F©Úbase_resultr   c                ó  • U R                  X#US9nUR                  UR                  5      nU R                  n	[	        U5      n
U
R                  UR                  5      n
U R                  X¨R                  5       XGS9nUR                  5       nX›-  R                  SS5      nU" U" U5      5      nSnUb  UR                  nUb  Xn-
  nO*[        R                  " U[        X R                  5      5      nUS-
  U-  XÍ-  U-  -   nU$ )úy
For DoRA, calculate the extra output from LoRA with DoRA applied. This should be added on top of the base layer
output.
©rV   rW   r   ©rJ   rK   rL   r   rC   r_   N)rY   rH   rI   rJ   r	   rN   Údetachri   ÚbiasÚFÚlinearr   r-   )r   ÚxrV   rW   rL   rm   rs   r   rK   Ú	magnituderJ   rM   Úmag_norm_scaleÚlora_resultry   Úresult_doras                   r   ÚforwardÚDoraLinearLayer.forward„   s  € ð
 ×*Ñ*°&ÐVbÐ*ÐcˆØ!—n‘n Q§W¡WÓ-ˆà—K‘Kˆ	Ü)¨*Ó5ˆØ—‘˜1Ÿ7™7Ó#ˆØ×*Ñ*Ø×'9Ñ'9Ó';ÀWð +ð 
ˆð "×(Ñ(Ó*ˆØ#Ñ1×7Ñ7¸¸2Ó>ˆá™V A›YÓ'ˆàˆØÑ"Ø—?‘?ˆDØÑØ)Ñ0�øäŸ(š( 1¤i°×8KÑ8KÓ&LÓMˆKà%¨Ñ)¨[Ñ8¸>Ñ;WÐZaÑ;aÑaˆØÐr!   c                 ó*   >• [         TU ]  5       nSU-   $ ©Nz
lora.dora.©r+   Ú__repr__©r   Úrepr0   s     €r   r†   ÚDoraLinearLayer.__repr__¨   ó   ø€ Ü‰gÑÓ ˆØ˜cÑ!Ð!r!   )r.   r-   rJ   )r4   N)Tr*   )re   Ú
__module__Ú__qualname__Ú__firstlineno__r,   Ústrr   r   r   r   r   Úboolr?   r%   rE   ÚTensorrN   rY   rp   r�   r†   Ú__static_attributes__Ú__classcell__©r0   s   @r   r'   r'   >   sß   ø† õ.ð
& ð &¨Cð &°Dô &ð2˜cð 2¨H°S©Mð 2¸hÀs¹mô 2ô!ñ˜$÷ ð ñ �]Ó#ñÈ(ÐSVÉ-ð Ðch×coÑcoô ó $ðñ �]Ó#ñ¸HÀS¹Mô ó $ðð QV÷ Dð> NRÐ`iõ "ðH"˜#÷ "õ "r!   r'   c                   ól   ^ • \ rS rSr\" S5      SS\\   4S jj5       rSS.S jrS\4U 4S	 jjr	S
r
U =r$ )ÚDoraEmbeddingLayeré­   rP   r   c                 ó   • X-  R                   $ r*   )rU   )r   rV   rW   r   s       r   rY   Ú"DoraEmbeddingLayer.get_lora_weight®   s   € à‘×"Ñ"Ð"r!   r7   )r   c                óÜ   • U R                  X#US9nU R                  n	UR                  n
U R                  X¨R                  5       XGS9nUR                  5       nX›-  nXÆ" X5      U-  -  U-  nXÍ4$ )ru   rv   rw   )rY   rJ   rN   rx   )r   r|   rV   rW   rL   rm   Úembed_fnr   rK   r}   rJ   rM   r~   r€   s                 r   r�   ÚDoraEmbeddingLayer.forward²   s‹   € ð
 ×*Ñ*°&ÐVbÐ*ÐcˆØ—K‘Kˆ	Ø×"Ñ"ˆØ×*Ñ*Ø×'9Ñ'9Ó';ÀWð +ð 
ˆð "×(Ñ(Ó*ˆØ"Ñ0ˆØ$¨°Ó(;¸fÑ(DÑEÈÑOˆØÐ*Ð*r!   r4   c                 ó*   >• [         TU ]  5       nSU-   $ r„   r…   r‡   s     €r   r†   ÚDoraEmbeddingLayer.__repr__È   rŠ   r!   r$   r*   )re   r‹   rŒ   r�   r%   r   rŽ   rY   r�   r†   r‘   r’   r“   s   @r   r•   r•   ­   sB   ø† Ù�]Ó#ñ#¸HÀS¹Mô #ó $ð#ð Ybõ +ð,"˜#÷ "õ "r!   r•   c                   óì   ^ • \ rS rSr\" S5      SS\\   S\R                  4S jj5       r	\" S5      SS\\   S\R                  4S jj5       r
SS	S
.S\S\R                  4S jjrS\4U 4S jjrSrU =r$ )Ú_DoraConvNdLayeréÍ   rA   Nr   r4   c                 óœ   • XU-  -   n[        [        SUR                  5       5      5      nUR                  SUSS9R	                  SS5      nU$ )NrC   é   T)ÚprD   Úkeepdimr   )ÚtupleÚrangerD   rG   r   )r   rJ   rK   rL   r   rD   rM   s          r   rN   Ú _DoraConvNdLayer.get_weight_normÎ   sO   € ð  KÑ/Ñ/ˆä”E˜!˜VŸZ™Z›\Ó*Ó+ˆØ—k‘k A¨3¸�kÐ=×GÑGÈÈ1ÓMˆØÐr!   rP   c                 óÒ   • UR                   R                  S   n[        R                  " UR                   R	                  SU/5      UR                   R	                  US/5      5      nU$ )Nr   r_   )rJ   rT   rE   rh   ri   )r   rV   rW   r   ro   rK   s         r   rY   Ú _DoraConvNdLayer.get_lora_weight×   sV   € ð �M‰M×Ñ Ñ"ˆÜ—h’h˜vŸ}™}×1Ñ1°2°q°'Ó:¸F¿M¹M×<NÑ<NÐPQÐSUÈwÓ<WÓXˆØÐr!   r7   rr   c          
      ó  • UR                   nU R                  X#US9R                  UR                  5      n	U R                   n
U R	                  X‰R                  5       XGS9nUR                  5       nX«-  nUc>  U R                  UUSUR                  UR                  UR                  UR                  S9nO9UR                  nUb*  SSUR                  5       S-
  -  -   nXmR                  " U6 -
  nUS-
  U-  XÃ" U" U5      5      -  U-  -   nU$ )	ru   rv   rw   N)ry   ÚstrideÚpaddingÚdilationÚgroups)rC   r_   )rC   r¢   rC   )rJ   rY   rj   rT   rN   rx   Úconv_fnr«   r¬   r­   r®   ry   rD   ri   )r   r|   rV   rW   rL   rm   rs   r   rJ   rK   r}   rM   r~   ry   Ú
bias_shaper€   s                   r   r�   Ú_DoraConvNdLayer.forwardß   s1  € ð ×"Ñ"ˆØ×*Ñ*°&ÐVbÐ*Ðc×kÑkØ�L‰Ló
ˆð —K‘Kˆ	Ø×*Ñ*Ø×'9Ñ'9Ó';ÀWð +ð 
ˆð "×(Ñ(Ó*ˆØ"Ñ0ˆàÑØŸ,™,ØØØØ!×(Ñ(Ø"×*Ñ*Ø#×,Ñ,Ø!×(Ñ(ð 'ð ‰Kð —?‘?ˆDØÑà$ t¨{¯©Ó/@À1Ñ/DÑ'EÑE�
Ø)¯IªI°zÐ,BÑB�à%¨Ñ)¨[Ñ8¸>ÈFÑSYÐZ[ÓS\ÓL]Ñ;]Ð`gÑ;gÑgˆØÐr!   c                 ó*   >• [         TU ]  5       nSU-   $ r„   r…   r‡   s     €r   r†   Ú_DoraConvNdLayer.__repr__  rŠ   r!   r$   r*   )re   r‹   rŒ   r�   r%   r   rŽ   rE   r�   rN   rY   r�   r†   r‘   r’   r“   s   @r   rŸ   rŸ   Í   s›   ø† Ù�]Ó#ñÈ(ÐSVÉ-ð Ðch×coÑcoô ó $ðñ �]Ó#ñ¸HÀS¹Mð ÐUZ×UaÑUaô ó $ðð FJÐ_hò*ØY\ð*à	�‰õ*ðX"˜#÷ "õ "r!   rŸ   c                   ó(   ^ • \ rS rSrU 4S jrSrU =r$ )ÚDoraConv1dLayeri  c                 óN   >• [         TU ]  U5        [        R                  U l        g r*   )r+   r,   rz   Úconv1dr¯   r/   s     €r   r,   ÚDoraConv1dLayer.__init__  ó   ø€ Ü‰Ñ˜Ô(Ü—x‘xˆ�r!   ©r¯   ©re   r‹   rŒ   r�   r,   r‘   r’   r“   s   @r   rµ   rµ     ó   ø† ÷ ó  r!   rµ   c                   ó(   ^ • \ rS rSrU 4S jrSrU =r$ )ÚDoraConv2dLayeri  c                 óN   >• [         TU ]  U5        [        R                  U l        g r*   )r+   r,   rz   Úconv2dr¯   r/   s     €r   r,   ÚDoraConv2dLayer.__init__  r¹   r!   rº   r»   r“   s   @r   r¾   r¾     r¼   r!   r¾   c                   ó(   ^ • \ rS rSrU 4S jrSrU =r$ )ÚDoraConv3dLayeri  c                 óN   >• [         TU ]  U5        [        R                  U l        g r*   )r+   r,   rz   Úconv3dr¯   r/   s     €r   r,   ÚDoraConv3dLayer.__init__  r¹   r!   rº   r»   r“   s   @r   rÃ   rÃ     r¼   r!   rÃ   )Úcopyr   Ú	functoolsr   Útypingr   r   rE   Útorch.nn.functionalr   Ú
functionalrz   Úpeft.utils.integrationsr	   r
   Úpeft.utils.otherr   r   rŽ   r%   ÚModuler'   r•   rŸ   rµ   r¾   rÃ   r$   r!   r   Ú<module>rÏ      s�   ðõ Ý ß  ã ß Ð Ý ç OÝ &ð Ð Ø að˜sô ô>l"�b—i‘iô l"ô^"˜ô "ô@@"�ô @"ôF Ð&ô  ô Ð&ô  ô Ð&õ  r!   