ó
    >:j�<  ã                   óò   • S SK r S SKJr  S SKrS SKJr  S SKJs  Jr  S SK	J
r
  S SKJrJr  S SKJr  SSKJr   " S S	\R$                  R&                  5      r " S
 S\5      r " S S\R,                  \5      rg)é    N)ÚOptional)ÚConv1D)ÚBaseTunerLayerÚcheck_adapters_to_merge)Ú	transposeé   )Ú
BufferDictc                   ó4   • \ rS rSr\S 5       r\S 5       rSrg)ÚUniqueBaseGradé   c                 óT   • US S 2S S 2S 4   U-  US   -  nU R                  XU5        U$ )N©N)Úsave_for_backward)ÚctxÚ
randlora_AÚrandlora_lambdaÚrandlora_gammaÚouts        ÚW/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/randlora/layer.pyÚforwardÚUniqueBaseGrad.forward   s6   € àša¢ D˜jÑ)¨JÑ6¸ÈÑ9NÑNˆØ×Ñ˜j¸>ÔJØˆ
ó    c                 ó(  • U R                   u  p#nUR                  UR                  5      UR                  UR                  5      UR                  UR                  5      pCn[        R                  " SXU5      n[        R                  " SXU5      nS XV4$ )Nzkbj,kvj,bj->kbzkbj,kvj,kb->bj)Úsaved_tensorsÚtoÚdtypeÚtorchÚeinsum)r   Úgrad_outputr   r   r   Úgrad_randlora_lambdaÚgrad_randlora_gammas          r   ÚbackwardÚUniqueBaseGrad.backward%   s‹   € à69×6GÑ6GÑ3ˆ
 ^à�M‰M˜+×+Ñ+Ó,Ø×Ñ˜{×0Ñ0Ó1Ø×Ñ˜k×/Ñ/Ó0ð &4ˆ
ô
  %Ÿ|š|Ð,<¸kÐWeÓfÐÜ#ŸlšlÐ+;¸[ÐVeÓfÐØÐ)Ð>Ð>r   © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ústaticmethodr   r"   Ú__static_attributes__r$   r   r   r   r      s(   † àñó ðð
 ñ	?ó ó	?r   r   c                   óx   • \ rS rSrSrSrS\R                  4S jr\	S\
4S j5       r SS\S	\S
\
4S jjrS rSrg)ÚRandLoraLayeré2   )r   r   )r   Ú
randlora_BÚ
base_layerc                 ó†  • Xl         0 U l        0 U l        [        R                  " 0 5      U l        [        R                  " 0 5      U l        [        R                  " 0 5      U l        S U l	        S U l
        SU l        / U l        SU l        U R                  5       n[        U[        R                   5      (       a  UR"                  UR$                  pCO^[        U[&        5      (       aI  [)        UR*                  S5      (       a  UR*                  R,                  OUR*                  R.                  u  p4WU l        WU l        X l        g )NFTÚds_shape)r/   ÚrÚscalingÚnnÚ
ModuleDictÚrandlora_dropoutÚParameterDictr   r   r   r.   Ú_disable_adaptersÚmerged_adaptersÚcast_input_dtype_enabledÚget_base_layerÚ
isinstanceÚLinearÚin_featuresÚout_featuresr   ÚhasattrÚweightr1   ÚshapeÚkwargs)Úselfr/   rC   r>   r?   s        r   Ú__init__ÚRandLoraLayer.__init__7   s  € Ø$ŒØˆŒØˆŒÜ "§¢¨bÓ 1ˆÔô  "×/Ò/°Ó3ˆÔÜ ×.Ò.¨rÓ2ˆÔð 15ˆŒØ04ˆŒð "'ˆÔØ!ˆÔð )-ˆÔ%à×(Ñ(Ó*ˆ
Ü�j¤"§)¡)×,Ñ,Ø(2×(>Ñ(>À
×@WÑ@W™Ü˜
¤F×+Ñ+ä.5°j×6GÑ6GÈ×.TÑ.T�
×!Ñ!×*Ò*ÐZd×ZkÑZk×ZqÑZqñ &ˆKð 'ˆÔØ(ˆÔØ�r   Úreturnc                 ó,   • [        U R                  5      $ r   )Úboolr9   )rD   s    r   ÚmergedÚRandLoraLayer.mergedY   s   € ä�D×(Ñ(Ó)Ð)r   r   r.   Úinference_modec	           	      óè  • US::  a  [        SU 35      eX@R                  U'   US:”  a  [        R                  " US9n
O[        R                  " 5       n
U R
                  R                  [        R                  " X05      5        [        U R                  U R                  5      U-  nUR                  5       (       a  [        U5      O[        U5      S-   U l        [        R                  " [        R                   " X@R                  5      SS9U R"                  U'   [        R                  " [        R$                  " U R                  [        U R                  U R                  5      5      ['        U R                  U R                  5      -  SS9U R(                  U'   XT-  U R*                  U'   X l        X0l        X;  Gaô  [1        U R,                  5      S:  a  [        S5      e[3        U R,                  R5                  5       5      S   n[3        U R.                  R5                  5       5      S   nS	n['        U R                  U R                  5      [        U R                  U R                  5      nnUR6                  S   U:  a)  [        UR9                  S
UR6                  S   U5      5      eUR6                  S   U:  a)  [        UR9                  SUR6                  S   U5      5      eSnUR6                  S   U R                  U   :  a6  [        UR9                  SUR6                  S   U R                  U   5      5      eUR6                  S   U R                  U   :  a6  [        UR9                  S
UR6                  S   U R                  U   5      5      eXÀR,                  U'   XÐR.                  U'   U(       a  U R;                  U5        U R=                  U5        U R?                  U R@                  US9  g )Nr   z?`r` should be a positive integer value but the value passed is ç        )Úpé   T)Úrequires_gradzfThe `randlora_A` and `randlora_B` buffers are empty. This should not happen. Please report this issue.z¨{} has a size of {} but {} or greater is required; this probably happened because an additional RandLora adapter was added after the first one with incompatible shapes.r.   éÿÿÿÿr   z×{} has a size of {} but {} or greater is required; this probably happened because an additional RandLora adapter with a lower rank was added after the first one; loading the adapters in reverse order may solve this.)rL   )!Ú
ValueErrorr2   r4   ÚDropoutÚIdentityr6   Úupdater5   Úminr>   r?   Ú
is_integerÚintÚ	num_basesÚ	Parameterr   Úrandnr   ÚonesÚmaxr   r3   r   r.   ÚlenÚlistÚvaluesrB   ÚformatÚreset_randlora_parametersÚ%_move_adapter_to_device_of_base_layerÚset_adapterÚactive_adapters)rD   Úadapter_namer   r.   r2   Úrandlora_alphar6   Úinit_weightsrL   rC   Úrandlora_dropout_layerrZ   Úrandlora_A_paramÚrandlora_B_paramÚ
error_tmplÚmax_dimÚmin_dims                    r   Úupdate_layerÚRandLoraLayer.update_layer]   sj  € ð �‹6ÜÐ^Ð_`Ð^aÐbÓcÐcØ �‰ˆ|ÑØ˜cÓ!Ü%'§Z¢ZÐ2BÑ%CÑ"ä%'§[¢[£]Ð"à×Ñ×$Ñ$¤R§]¢]°LÐ3YÓ%ZÔ[ô ˜×(Ñ(¨$×*;Ñ*;Ó<¸qÑ@ˆ	Ø+4×+?Ñ+?×+AÑ+Aœ˜YœÄsÈ9Ã~ÐXYÑGYˆŒÜ-/¯\ª\¼%¿+º+ÀaÏÉÓ:XÐhlÑ-mˆ×Ñ˜\Ñ*Ü,.¯LªLÜ�JŠJ�t—~‘~¤s¨4×+<Ñ+<¸d×>NÑ>NÓ'OÓPÜ�$×#Ñ# T×%5Ñ%5Ó6ñ7àñ-
ˆ×Ñ˜LÑ)ð &4Ñ%7ˆ�‰�\Ñ"ð %ŒØ$ŒØÔ)ä�4—?‘?Ó# aÓ'Ü Ø|óð ô  $ D§O¡O×$:Ñ$:Ó$<Ó=¸aÑ@ÐÜ# D§O¡O×$:Ñ$:Ó$<Ó=¸aÑ@ÐðRð ô  # 4×#3Ñ#3°T×5FÑ5FÓGÌÈT×M]ÑM]Ð_c×_pÑ_pÓIq�WˆGà×%Ñ% aÑ(¨7Ó2Ü  ×!2Ñ!2°<ÐAQ×AWÑAWÐXYÑAZÐ\cÓ!dÓeÐeà×%Ñ% bÑ)¨GÓ3Ü  ×!2Ñ!2°<ÐAQ×AWÑAWÐXYÑAZÐ\cÓ!dÓeÐeð3ð ð
  ×%Ñ% aÑ(¨4¯6©6°,Ñ+?Ó?Ü  ×!2Ñ!2°<ÐAQ×AWÑAWÐXYÑAZÐ\`×\bÑ\bÐcoÑ\pÓ!qÓrÐrà×%Ñ% bÑ)¨D¯F©F°<Ñ,@Ó@Ü  ×!2Ñ!2°<ÐAQ×AWÑAWÐXZÑA[Ð]a×]cÑ]cÐdpÑ]qÓ!rÓsÐsà,<�O‰O˜LÑ)Ø,<�O‰O˜LÑ)æØ×*Ñ*¨<Ô8à×2Ñ2°<Ô@Ø×Ñ˜×-Ñ-¸nÐÒMr   c           	      ó–  • XR                   R                  5       ;   a›  [        R                  " 5          [        R
                  R                  U R                   U   5        [        R
                  R                  U R                  U   S[        U R                  U   R                  5      -  5        S S S 5        g g ! , (       d  f       g = f)NrP   )r   Úkeysr   Úno_gradr4   ÚinitÚzeros_Ú	constant_r   r^   rB   ©rD   rg   s     r   rc   Ú'RandLoraLayer.reset_randlora_parameters­   sˆ   € Ø×/Ñ/×4Ñ4Ó6Ó6Ü—’•Ü—‘—‘˜t×3Ñ3°LÑAÔBÜ—‘×!Ñ! $×"5Ñ"5°lÑ"CÀQÌÈT×M`ÑM`ÐamÑMn×MtÑMtÓIuÑEuÔv÷ !�ð 7ß •ús   ³A=B:Â:
C)r8   r/   r:   r>   rC   r9   rZ   r?   r2   r   r.   r6   r   r   r3   N)F)r%   r&   r'   r(   Úadapter_layer_namesÚother_param_namesr4   ÚModulerE   ÚpropertyrI   rJ   r	   rp   rc   r*   r$   r   r   r,   r,   2   sq   † à?ÐØ4Ðð  2§9¡9ô  ðD ð*˜ó *ó ð*ð  %ñNNð ðNNð ð	NNð õNNõ`wr   r,   c                   ó^  ^ • \ rS rSr      SS\S\S\S\S\S\S\S	\S
\SS4U 4S jjjr	SS\S\
\\      SS4S jjrSS jrSS\\R                   \R                   4   4S jjrS\R                   4S jrS\R                   S\R                   4S jrSS\S\4S jjrS\4U 4S jjrSrU =r$ )r=   é´   r   r.   rg   r2   rh   r6   Úfan_in_fan_outÚis_target_conv_1d_layerri   rG   Nc           	      ó¸   >• [         [        R                  U ]  5         [        R                  " X40 UD6  X€l        X@l        U R                  XBX5XgU
5        X�l        g r   )	Úsuperr4   r=   rE   r,   r€   Ú_active_adapterrp   r�   )rD   r/   r   r.   rg   r2   rh   r6   r€   r�   ri   rC   Ú	__class__s               €r   rE   ÚLinear.__init__¶   sP   ø€ ô 	Œb�i‰i˜Ñ'Ô)Ü×Ò˜tÑ:°6Ò:Ø,ÔØ+ÔØ×Ñ˜,°JÀ>ÐeqÔrØ'>Õ$r   Ú
safe_mergeÚadapter_namesc                 ó¬  • [        X5      nU(       d  gU GH:  nX0R                  R                  5       ;   d  M#  U R                  5       nUR                  R
                  nU(       a�  UR                  R                  R                  5       nX`R                  U5      -  n[        R                  " U5      R                  5       (       d  [        SU S35      eUR                  U5      UR                  l        O?U R                  U5      nUR                  =R                  UR                  U5      -  sl        U R                  R                  U5        GM=     g)a  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`list[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
        to `None`.
Nz1NaNs detected in the merged weights. The adapter z seems to be broken)r   r   rs   r;   rA   r   ÚdataÚcloneÚget_delta_weightr   ÚisfiniteÚallrS   r   r9   Úappend)rD   r‡   rˆ   Úactive_adapterr/   Ú
orig_dtypeÚorig_weightsÚdelta_weights           r   ÚmergeÚLinear.mergeÌ   s  € ô 0°ÓDˆÞàä+ˆNØ×!5Ñ!5×!:Ñ!:Ó!<Õ<Ø!×0Ñ0Ó2�
Ø'×.Ñ.×4Ñ4�
æð $.×#4Ñ#4×#9Ñ#9×#?Ñ#?Ó#A�Là ×$9Ñ$9¸.Ó$IÑI�Lä Ÿ>š>¨,Ó7×;Ñ;×=Ñ=Ü(ØOÐP^ÐO_Ð_rÐsóð ð .:¯_©_¸ZÓ-H�J×%Ñ%Õ*à#'×#8Ñ#8¸Ó#H�LØ×%Ñ%×*Ò*¨l¯o©o¸jÓ.IÑIÕ*à×$Ñ$×+Ñ+¨N×;ò- ,r   c                 óö  • U R                   (       d  [        R                  " S5        g[        U R                  5      S:”  a¸  U R                  5       nUR                  R                  nU R                  R                  5       nX0R                  R                  5       ;   a?  U R                  U5      nUR                  =R                  UR                  U5      -  sl        [        U R                  5      S:”  a  M·  gg)zG
This method unmerges all merged adapter layers from the base weights.
z Already unmerged. Nothing to do.Nr   )rJ   ÚwarningsÚwarnr_   r9   r;   rA   r   Úpopr   rs   rŒ   rŠ   r   )rD   r/   r‘   r�   r“   s        r   ÚunmergeÚLinear.unmergeö   s»   € ð �{�{Ü�MŠMÐ<Ô=Øä�$×&Ñ&Ó'¨!Ó+Ø×,Ñ,Ó.ˆJØ#×*Ñ*×0Ñ0ˆJØ!×1Ñ1×5Ñ5Ó7ˆNØ×!5Ñ!5×!:Ñ!:Ó!<Ó<Ø#×4Ñ4°^ÓD�Ø×!Ñ!×&Ò&¨,¯/©/¸*Ó*EÑEÕ&ô �$×&Ñ&Ó'¨!×+r   c                 óÒ  • U R                   U   nU R                  U   nUc  UR                  nUR                  nUR                  S:H  =(       a-    U[
        R                  :H  =(       d    U[
        R                  :H  nU R                  U   R                  U5      nU R                  U   R                  U5      nU(       a@  UR                  5       nUR                  5       nUR                  5       nUR                  5       n[        U R                  U R                  5      [        U R                  U R                  5      p©USS2SU R                   2SU	24   R                  U5      nUSU
2SU R                   2SS24   R                  U5      nUR#                  SS9n[$        R'                  X·U5      R#                  SS9nX�R                  :X  a  Xí4$ UR(                  UR(                  4$ )a  
Performs scaling on the smallest random base (randlora_A) and returns randlora_A and randlora_B in the correct
order to fit the target layers' dimensions

Args:
    adapter (str):
        The name of the adapter for which the delta weight should be computed.
NÚcpurP   )Ú	start_dim)Úend_dim)r   r.   Údevicer   Útyper   Úfloat16Úbfloat16r   r   r   ÚfloatrW   r?   r>   r^   rZ   Úflattenr   ÚapplyÚT)rD   Úadapterr    r   r.   r   Úcast_to_fp32r   r   ro   rn   Úsliced_AÚsliced_BÚupdate_BÚupdate_As                  r   Úget_scaled_basesÚLinear.get_scaled_bases  sÁ  € ð —_‘_ WÑ-ˆ
Ø—_‘_ WÑ-ˆ
Ø‰>Ø×&Ñ&ˆFØ× Ñ ˆð
 —{‘{ eÑ+×c°¼%¿-¹-Ñ1G×1bÈ5ÔTY×TbÑTbÑKbˆà×.Ñ.¨wÑ7×:Ñ:¸6ÓBˆØ×,Ñ,¨WÑ5×8Ñ8¸Ó@ˆæØ#×)Ñ)Ó+ˆJØ#×)Ñ)Ó+ˆJØ-×3Ñ3Ó5ˆOØ+×1Ñ1Ó3ˆNô ˜t×0Ñ0°$×2BÑ2BÓCÄSÈ×IZÑIZÐ\`×\lÑ\lÓEm�ð
 šaÐ!1 4§>¡>Ð!1°8°G°8Ð;Ñ<×?Ñ?ÀÓGˆØ˜h˜w˜hÐ(8¨$¯.©.Ð(8º!Ð;Ñ<×?Ñ?ÀÓGˆð ×#Ñ#¨aÐ#Ð0ˆÜ!×'Ñ'¨À>ÓR×ZÑZÐcdÐZÐeˆð ×&Ñ&Ó&ØÐ%Ð%Ø�z‰z˜8Ÿ:™:Ð%Ð%r   c                 ó¾   • U R                  U5      u  p#UR                  UR                  -  R                  n[        X@R                  5      nU R                  U   nXV-  $ )zš
Compute the delta weight for the given adapter.

Args:
    adapter (str):
        The name of the adapter for which the delta weight should be computed.
)r®   r§   r   r€   r3   )rD   r¨   r¬   r­   rV   Úoutput_tensorr3   s          r   rŒ   ÚLinear.get_delta_weight6  sV   € ð "×2Ñ2°7Ó;Ñˆà—*‘*˜xŸz™zÑ)×,Ñ,ˆÜ! &×*=Ñ*=Ó>ˆà—,‘,˜wÑ'ˆØÑ&Ð&r   Úxc           	      óÊ  • UR                   nU R                  (       a8  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nOüU R                  (       a  U R                  " U/UQ70 UD6nOÔU R                  " U/UQ70 UD6nU R
                   H®  nX`R                  R                  5       ;  a  M"  U R                  U   nU R                  XaR                  S9u  p‰UR                  U	R                   5      nU R                  U   n
U[        R                  " [        R                  " U" U5      U5      U	5      U
-  -   nM°     UR                  U5      nU$ )N)r    )r   Údisable_adaptersrJ   rš   r/   rf   r   rs   r6   r®   r    r   r3   ÚFÚlinear)rD   r³   ÚargsrC   Úprevious_dtypeÚresultr�   Údropoutr¬   r­   r3   s              r   r   ÚLinear.forwardG  s&  € ØŸ™ˆØ× × Ø�{�{Ø—‘”Ø—_’_ QÐ8¨Ò8°Ñ8‰FØ�[�[Ø—_’_ QÐ8¨Ò8°Ñ8‰Fà—_’_ QÐ8¨Ò8°Ñ8ˆFØ"&×"6Ô"6�Ø!×)=Ñ)=×)BÑ)BÓ)DÓDÙØ×/Ñ/°Ñ?�Ø%)×%:Ñ%:¸>×RZÑRZÐ%:Ð%[Ñ"�Ø—D‘D˜Ÿ™Ó(�ØŸ,™, ~Ñ6�Ø¤!§(¢(¬1¯8ª8±G¸A³JÀÓ+IÈ8Ó"TÐW^Ñ"^Ñ^’ñ #7ð —‘˜>Ó*ˆØˆr   c                 ó   • g)NTr$   rx   s     r   Úsupports_lora_conversionÚLinear.supports_lora_conversion\  s   € Ør   c                 ó*   >• [         TU ]  5       nSU-   $ )Nz	randlora.)rƒ   Ú__repr__)rD   Úrepr…   s     €r   rÁ   ÚLinear.__repr___  s   ø€ Ü‰gÑÓ ˆØ˜SÑ Ð r   )r„   r€   r�   )r   r   rN   FFT)FN)rG   Nr   )Údefault)r%   r&   r'   r(   r	   ÚstrrY   r¤   rI   rE   r   r`   r”   rš   Útupler   ÚTensorr®   rŒ   r   r¾   rÁ   r*   Ú__classcell__)r…   s   @r   r=   r=   ´   s  ø† ð ØØ"%Ø$Ø(-Ø!ñ?ð ð?ð ð	?ð
 ð?ð ð?ð ð?ð  ð?ð ð?ð "&ð?ð ð?ð 
÷?ð ?ñ,(< ð (<¸XÀdÈ3ÁiÑ=Pð (<Ð\`õ (<ôTFñ .&¸¸e¿l¹lÈEÏLÉLÐ>XÑ8Yõ .&ð`'¨5¯<©<ô 'ð"˜Ÿ™ð ¸5¿<¹<ô ñ*°Sð Èõ ð!˜#÷ !õ !r   r=   )r—   Útypingr   r   Útorch.nnr4   Útorch.nn.functionalÚ
functionalr¶   Útransformers.pytorch_utilsr   Úpeft.tuners.tuners_utilsr   r   Úpeft.utils.otherr   Ú_buffer_dictr	   ÚautogradÚFunctionr   r,   r=   r$   r   r   Ú<module>rÓ      s`   ðó Ý ã Ý ß Ð Ý -ç LÝ &å %ô?�U—^‘^×,Ñ,ô ?ô*w�Nô wôDm!ˆR�Y‰Y˜õ m!r   