ó
    >:jÌ+  ã                   ó¨   • S SK r S SKJr  S SKrS SKJr  S SKJs  Jr  S SK	J
r
  S SKJrJr  S SKJr   " S S\5      r " S S	\R"                  \5      rg)
é    N)ÚOptional)ÚConv1D)ÚBaseTunerLayerÚcheck_adapters_to_merge)Ú	transposec                   óŒ   • \ rS rSrSrS\R                  4S jr\S\	4S j5       r
   SS\S\S	\S
\S\S\S\S\	4S jjrS rSrg)ÚVBLoRALayeré   )Úvblora_logits_AÚvblora_logits_BÚvblora_vector_bankÚ
base_layerc                 ó\  • Xl         0 U l        0 U l        [        R                  " 0 5      U l        [        R                  " 0 5      U l        [        R                  " 0 5      U l        SU l	        / U l
        U R                  5       n[        U[        R                  5      (       a  UR                  UR                  pCO^[        U[         5      (       aI  [#        UR$                  S5      (       a  UR$                  R&                  OUR$                  R(                  u  p4WU l        WU l        X l        g )NFÚds_shape)r   ÚrÚtopkÚnnÚ
ModuleDictÚvblora_dropoutÚParameterDictr   r   Ú_disable_adaptersÚmerged_adaptersÚget_base_layerÚ
isinstanceÚLinearÚin_featuresÚout_featuresr   ÚhasattrÚweightr   ÚshapeÚkwargs)Úselfr   r!   r   r   s        ÚU/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/vblora/layer.pyÚ__init__ÚVBLoRALayer.__init__   sí   € Ø$ŒØˆŒØˆŒ	Ü Ÿmšm¨BÓ/ˆÔô  "×/Ò/°Ó3ˆÔÜ!×/Ò/°Ó3ˆÔð "'ˆÔØ!ˆÔà×(Ñ(Ó*ˆ
Ü�j¤"§)¡)×,Ñ,Ø(2×(>Ñ(>À
×@WÑ@W™Ü˜
¤F×+Ñ+ä.5°j×6GÑ6GÈ×.TÑ.T�
×!Ñ!×*Ò*ÐZd×ZkÑZk×ZqÑZqñ &ˆKð 'ˆÔØ(ˆÔØ�ó    Úreturnc                 ó,   • [        U R                  5      $ ©N)Úboolr   )r"   s    r#   ÚmergedÚVBLoRALayer.merged9   s   € ä�D×(Ñ(Ó)Ð)r&   Úadapter_namer   r   Únum_vectorsÚvector_lengthr   Úinit_logits_stdÚinference_modec
                 óœ  • US::  a  [        SU S35      eUS::  a  [        SU S35      eU R                  U-  S:w  a  [        SU R                   SU 35      eU R                  U-  S:w  a  [        SU R                   SU 35      eX0R                  U'   X@R                  U'   US:”  a  [
        R                  " US	9nO[
        R                  " 5       nU R                  R                  [
        R                  " X05      5        [
        R                  " [        R                  " X0R                  U-  U5      S
S9U R                  U'   [
        R                  " [        R                  " U R                  U-  X55      S
S9U R                  U'   X l        U R#                  X5        U R%                  U5        U R'                  U R(                  U	S9  g )Nr   z`r` z# should be a positive integer valuez`topk` z`in_features` z& must be divisible by `vector_length` z`out_features` ç        )ÚpT)Úrequires_grad)r1   )Ú
ValueErrorr   r   r   r   r   ÚDropoutÚIdentityr   Úupdater   Ú	ParameterÚtorchÚzerosr   r   r   Úreset_vblora_logitsÚ%_move_adapter_to_device_of_base_layerÚset_adapterÚactive_adapters)r"   r-   r   r   r   r.   r/   r   r0   r1   r!   Úvblora_dropout_layers               r#   Úupdate_layerÚVBLoRALayer.update_layer=   s±  € ð �‹6Ü˜t A 3Ð&IÐJÓKÐKØ�1‹9Ü˜w t fÐ,OÐPÓQÐQà×Ñ˜mÑ+¨qÓ0Ü˜~¨d×.>Ñ.>Ð-?Ð?eÐfsÐetÐuÓvÐvØ×Ñ˜}Ñ,°Ó1ÜØ! $×"3Ñ"3Ð!4Ð4ZÐ[hÐZiÐjóð ð  !�‰ˆ|ÑØ"&�	‰	�,ÑØ˜CÓÜ#%§:¢:°Ñ#?Ñ ä#%§;¢;£=Ð Ø×Ñ×"Ñ"¤2§=¢=°,Ð1UÓ#VÔWÜ-/¯\ª\Ü�KŠK˜×+Ñ+¨}Ñ<¸kÓJÐZ^ñ.
ˆ×Ñ˜\Ñ*ô .0¯\ª\Ü�KŠK˜×)Ñ)¨]Ñ:¸AÓKÐ[_ñ.
ˆ×Ñ˜\Ñ*ð #5ÔØ× Ñ  Ô?Ø×2Ñ2°<Ô@Ø×Ñ˜×-Ñ-¸nÐÒMr&   c                 óV  • XR                   R                  5       ;   a{  [        R                  " 5          [        R
                  R                  U R                   U   SU5        [        R
                  R                  U R                  U   SU5        S S S 5        g g ! , (       d  f       g = f)Nr   )r   Úkeysr;   Úno_gradr   ÚinitÚnormal_r   )r"   r-   r0   s      r#   r=   ÚVBLoRALayer.reset_vblora_logitsh   sp   € Ø×/Ñ/×4Ñ4Ó6Ó6Ü—’•Ü—‘—‘ × 4Ñ 4°\Ñ BÀAÀÔWÜ—‘—‘ × 4Ñ 4°\Ñ BÀAÀÔW÷ !�ð 7ß •ús   ³ABÂ
B()r   r   r   r!   r   r   r   r   r   r   r   r   N)r3   ç{®Gáz„?F)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Úadapter_layer_namesr   ÚModuler$   Úpropertyr*   r+   ÚstrÚintÚfloatrB   r=   Ú__static_attributes__© r&   r#   r	   r	      s¡   † àVÐð 2§9¡9ô ð4 ð*˜ó *ó ð*ð !$Ø!%Ø$ñ)Nàð)Nð ð	)Nð
 ð)Nð ð)Nð ð)Nð ð)Nð ð)Nð õ)NõVXr&   r	   c                   ó–  ^ • \ rS rSr     SS\S\S\S\S\S\S\S	\S
\SS4U 4S jjjrSS\S\	\
\      SS4S jjrSS jrS\R                  S\R                  4S jrSS\\R                  \R                  4   4S jjrS\R                  4S jrS\R                  S\R                  4S jrSS\S\4S jjrS\4U 4S jjrSrU =r$ ) r   éo   r-   r   r.   r/   r   r   r0   Úfan_in_fan_outÚis_target_conv_1d_layerr'   Nc           
      ó¸   >• [         [        R                  U ]  5         [        R                  " X40 UD6  X l        X0l        U R                  X2XGXVX‰5        X°l        g r)   )	Úsuperr   r   r$   r	   rY   Ú_active_adapterrB   rZ   )r"   r   r   r-   r   r.   r/   r   r   r0   rY   rZ   r!   Ú	__class__s                €r#   r$   ÚLinear.__init__q   sU   ø€ ô  	Œb�i‰i˜Ñ'Ô)Ü×Ò˜TÑ8°Ò8Ø,ÔØ+ÔØ×ÑØ¨a°{ÐSaô	
ð (?Õ$r&   Ú
safe_mergeÚadapter_namesc                 ó>  • [        X5      nU(       d  gU GH  nX0R                  R                  5       ;   d  M#  U R                  5       nU(       a€  UR                  R
                  R                  5       nXPR                  U5      -  n[        R                  " U5      R                  5       (       d  [        SU S35      eXTR                  l        O.UR                  =R
                  U R                  U5      -  sl        U R                  R                  U5        GM     g)a  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`List[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
        to `None`.
Nz1NaNs detected in the merged weights. The adapter z seems to be broken)r   r   rE   r   r   ÚdataÚcloneÚget_delta_weightr;   ÚisfiniteÚallr6   r   Úappend)r"   r`   ra   Úactive_adapterr   Úorig_weightss         r#   ÚmergeÚLinear.mergeŠ   së   € ô 0°ÓDˆÞàä+ˆNØ×!5Ñ!5×!:Ñ!:Ó!<Õ<Ø!×0Ñ0Ó2�
Þð $.×#4Ñ#4×#9Ñ#9×#?Ñ#?Ó#A�LØ ×$9Ñ$9¸.Ó$IÑI�LÜ Ÿ>š>¨,Ó7×;Ñ;×=Ñ=Ü(ØOÐP^ÐO_Ð_rÐsóð ð .:×%Ñ%Õ*à×%Ñ%×*Ò*¨d×.CÑ.CÀNÓ.SÑSÕ*Ø×$Ñ$×+Ñ+¨N×;ò ,r&   c                 ó¤  • U R                   (       d  [        R                  " S5        g [        U R                  5      S:”  a�  U R                  R                  5       nXR                  R                  5       ;   a<  U R                  5       R                  =R                  U R                  U5      -  sl
        [        U R                  5      S:”  a  MŽ  g g )Nz Already unmerged. Nothing to do.r   )r+   ÚwarningsÚwarnÚlenr   Úpopr   rE   r   r   rc   re   )r"   ri   s     r#   ÚunmergeÚLinear.unmerge­   s•   € Ø�{�{Ü�MŠMÐ<Ô=Øä�$×&Ñ&Ó'¨!Ó+Ø!×1Ñ1×5Ñ5Ó7ˆNØ×!5Ñ!5×!:Ñ!:Ó!<Ó<Ø×#Ñ#Ó%×,Ñ,×1Ò1°T×5JÑ5JÈ>Ó5ZÑZÕ1ô �$×&Ñ&Ó'¨!×+r&   Úlogitsc                 óš   • UR                  USS9u  pE[        R                  " USS9nUR                  S5      X%   -  R	                  S5      $ )Néÿÿÿÿ)Údiméþÿÿÿ)r   ÚFÚsoftmaxÚ	unsqueezeÚsum)r"   rt   r   r   Útop_k_logitsÚindicesÚtopk_weightss          r#   Ú_get_low_rank_matrixÚLinear._get_low_rank_matrix·   sL   € Ø &§¡¨D°b Ð 9ÑˆÜ—y’y °2Ñ6ˆØ×&Ñ& rÓ*Ð-?Ñ-HÑH×MÑMÈbÓQÐQr&   c                 ó€  • U R                   U   nU R                  U   nU R                  (       a1  US   R                  5       R	                  5       (       a  [        S5      eU R                  U   R                  UR                  5      nU R                  U   nU(       a0  UR                  5       nUR                  5       nUR                  5       nU R                  X5U5      R                  UR                  S   S5      nU R                  XEU5      R                  SS5      R                  SUR                  S   5      nXx4$ )N)r   r   zoFound infinity values in VB-LoRA logits. Ensure training was not resumed from a `save_only_topk_weights` model.r   rv   é   é   )r   r   ÚtrainingÚisinfÚanyÚRuntimeErrorr   ÚtoÚdevicer   rT   r€   Úreshaper    r   )	r"   ÚadapterÚcast_to_fp32r   r   r   r   ÚAÚBs	            r#   Ú_get_lora_matricesÚLinear._get_lora_matrices¼   s-  € Ø×.Ñ.¨wÑ7ˆØ×.Ñ.¨wÑ7ˆð �=�=˜_¨TÑ2×8Ñ8Ó:×>Ñ>×@Ñ@Üð Bóð ð "×4Ñ4°WÑ=×@Ñ@À×AWÑAWÓXÐØ�y‰y˜Ñ!ˆö Ø-×3Ñ3Ó5ˆOØ-×3Ñ3Ó5ˆOØ!3×!9Ñ!9Ó!;Ðð ×%Ñ% oÈ4ÓP×XÑXÐYh×YnÑYnÐopÑYqÐsuÓvˆð ×%Ñ% oÈ4ÓPß‰Y�q˜!‹_ß‰W�R˜×.Ñ.¨qÑ1Ó2ð 	
ð
 ˆtˆr&   c                 ó  • U R                   U   R                  nU R                   U   R                  nUR                  S:H  =(       a    U[        R
                  :H  nU R                  X5      u  pV[        Xe-  U R                  5      nU$ )zš
Compute the delta weight for the given adapter.

Args:
    adapter (str):
        The name of the adapter for which the delta weight should be computed.
Úcpu)	r   rŠ   ÚdtypeÚtyper;   Úfloat16r�   r   rY   )r"   rŒ   rŠ   r”   r�   rŽ   r�   Úoutput_tensors           r#   re   ÚLinear.get_delta_weightÚ   sx   € ð ×%Ñ% gÑ.×5Ñ5ˆØ×$Ñ$ WÑ-×3Ñ3ˆØ—{‘{ eÑ+×F°¼¿¹Ñ0FˆØ×&Ñ& wÓ=‰ˆÜ! !¡%¨×)<Ñ)<Ó=ˆØÐr&   Úxc           	      ó°  • UR                   nU R                  (       a8  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nOïU R                  (       a  U R                  " U/UQ70 UD6nOÇU R                  " U/UQ70 UD6nU R
                   H¡  nX`R                  R                  5       ;  a  M"  U R                  U5      u  pxUR                  U R                  U   R                   5      nU R                  U   n	U[        R                  " [        R                  " U	" U5      U5      U5      -   nM£     UR                  U5      nU$ r)   )r”   Údisable_adaptersr+   rr   r   r@   r   rE   r�   r‰   r   r   ry   Úlinear)
r"   r™   Úargsr!   Úprevious_dtypeÚresultri   rŽ   r�   Údropouts
             r#   ÚforwardÚLinear.forwardé   s  € ØŸ™ˆØ× × Ø�{�{Ø—‘”Ø—_’_ QÐ8¨Ò8°Ñ8‰FØ�[�[Ø—_’_ QÐ8¨Ò8°Ñ8‰Fà—_’_ QÐ8¨Ò8°Ñ8ˆFØ"&×"6Ô"6�Ø!×)=Ñ)=×)BÑ)BÓ)DÓDÙØ×.Ñ.¨~Ó>‘�Ø—D‘D˜×0Ñ0°Ñ@×FÑFÓG�Ø×-Ñ-¨nÑ=�Ø¤!§(¢(¬1¯8ª8±G¸A³JÀÓ+BÀAÓ"FÑF’ñ #7ð —‘˜>Ó*ˆØˆr&   c                 ó   • g)NTrV   )r"   r-   s     r#   Úsupports_lora_conversionÚLinear.supports_lora_conversioný   s   € Ør&   c                 ó*   >• [         TU ]  5       nSU-   $ )Nzvblora.)r\   Ú__repr__)r"   Úrepr^   s     €r#   r§   ÚLinear.__repr__   s   ø€ Ü‰gÑÓ ˆØ˜3‰Ðr&   )r]   rY   rZ   )r„   r3   rJ   FF)FN)r'   N)F)Údefault)rK   rL   rM   rN   rR   rS   rT   r*   r$   r   Úlistrk   rr   r;   ÚtensorÚTensorr€   Útupler�   re   r¡   r¤   r§   rU   Ú__classcell__)r^   s   @r#   r   r   o   s>  ø† ð Ø #Ø!%Ø$Ø(-ñ?ð ð	?ð
 ð?ð ð?ð ð?ð ð?ð ð?ð ð?ð ð?ð "&ð?ð 
÷?ð ?ñ2!< ð !<¸XÀdÈ3ÁiÑ=Pð !<Ð\`õ !<ôF[ðR¨5¯<©<ð RÐV[×VbÑVbô Rñ
ÀÀuÇ|Á|ÐUZ×UaÑUaÐGaÑAbõ ð<¨5¯<©<ô ð˜Ÿ™ð ¸5¿<¹<ô ñ(°Sð Èõ ð˜#÷ õ r&   r   )rn   Útypingr   r;   Útorch.nnr   Útorch.nn.functionalÚ
functionalry   Útransformers.pytorch_utilsr   Úpeft.tuners.tuners_utilsr   r   Úpeft.utils.otherr   r	   r   rV   r&   r#   Ú<module>r·      sG   ðó Ý ã Ý ß Ð Ý -ç LÝ &ôQX�.ô QXôhSˆR�Y‰Y˜õ Sr&   