ó
    >:j÷¾  ã                  óv  • S SK Jr  S SKrS SKJrJrJr  S SKrS SKJ	r	  S SK
J	s  Jr  S SKJrJr  SSKJr   " S S\	R$                  5      r " S	 S
\	R$                  5      r " S S\5      r " S S\	R$                  \5      r " S S\	R$                  \5      r " S S\	R$                  \5      r        SS jrg)é    )ÚannotationsN)ÚAnyÚOptionalÚUnion)ÚBaseTunerLayerÚcheck_adapters_to_mergeé   )Ú	OFTConfigc                  ó6   ^ • \ rS rSrSrSU 4S jjrS rSrU =r$ )ÚMultiplicativeDropoutLayeré   z6
Implements the multiplicative dropout layer for OFT.
c                ó.   >• [         TU ]  5         Xl        g)z�
Initializes the multiplicative dropout layer.

Parameters:
p (float): The probability of dropping out a block. Defaults to 0.0.
N)ÚsuperÚ__init__Úp)Úselfr   Ú	__class__s     €ÚR/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/oft/layer.pyr   Ú#MultiplicativeDropoutLayer.__init__!   s   ø€ ô 	‰ÑÔØ�ó    c                ót  • U R                   (       Ga%  U R                  S:”  Ga  UR                  S   UR                  S   :w  a  [        S5      eUR                  u  p#nUS:X  a  U$ [	        U R                  U-  5      nX%-
  n[
        R                  " [
        R                  " XQR                  S9[
        R                  " XaR                  S9/5      nU[
        R                  " U5         R                  USS5      n[
        R                  " X1R                  S9R                  USS5      nSU-
  U-  Xx-  -   nU$ )a  
Applies multiplicative dropout to the input tensor.

Parameters:
x (Tensor): The input tensor of shape (D, H, H), where `D` represents
            the number of OFT blocks, and `H` is the size of the square blocks along the last two dimensions,
            the block size in OFT.
r   éÿÿÿÿéþÿÿÿz4The last two dimensions of input should be the same!r	   ©Údevice)Útrainingr   ÚshapeÚ
ValueErrorÚintÚtorchÚcatÚonesr   ÚzerosÚrandpermÚviewÚeyeÚrepeat)	r   ÚxÚDÚHÚ_Únum_to_replaceÚ	num_zerosÚmaskÚ
eye_matrixs	            r   ÚforwardÚ"MultiplicativeDropoutLayer.forward+   sü   € ð �=�=ˆ=˜TŸV™V aœZà�w‰w�r‰{˜aŸg™g b™kÓ)Ü Ð!WÓXÐXà—g‘g‰GˆA�!ð �A‹vØ�ä  §¡¨!¡›_ˆNØÑ*ˆIÜ—9’9œeŸjšj¨ÇÁÑIÌ5Ï;Ê;ÐW`×iqÑiqÑKrÐsÓtˆDØœŸš qÓ)Ñ*×/Ñ/°°1°aÓ8ˆDÜŸš 1¯X©XÑ6×=Ñ=¸aÀÀAÓFˆJØ�T‘˜Q‘ Ñ!2Ñ2ˆAØˆr   ©r   )ç        )	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r0   Ú__static_attributes__Ú__classcell__©r   s   @r   r   r      s   ø† ñ÷÷ð r   r   c                  ó’   ^ • \ rS rSr      SU 4S jjrS rS r S         SS jjrSS jrSS jr	S r
S	 rS
 rS rSrU =r$ )ÚOFTRotationModuleéH   c                ón  >• [         TU ]  5         Xl        X l        X0l        X@l        [        R                  " [        R                  " X5      5      U l
        XPl        X`l        Xpl        X€l        X�l        X l        [        R"                  " X3S5      u  p¼U R%                  SUSS9  U R%                  SUSS9  g )Nr	   ÚrowsF)Ú
persistentÚcols)r   r   ÚrÚ
n_elementsÚ
block_sizeÚin_featuresÚnnÚ	Parameterr    ÚemptyÚweightÚcoftÚepsÚblock_shareÚkernel_sizeÚuse_cayley_neumannÚnum_cayley_neumann_termsÚtriu_indicesÚregister_buffer)r   rC   rD   rE   rF   rK   rL   rM   rN   rO   rP   r@   rB   r   s                €r   r   ÚOFTRotationModule.__init__I   sž   ø€ ô 	‰ÑÔØŒØ$ŒØ$ŒØ&ÔÜ—l’l¤5§;¢;¨qÓ#=Ó>ˆŒØŒ	ØŒØ&Ôà&ÔØ"4ÔØ(@Ô%ä×'Ò'¨
ÀÓB‰
ˆØ×Ñ˜V T°eÐÑ<Ø×Ñ˜V T°eÐÒ<r   c                óÚ   • UR                   S   n[        R                  " X2X!R                  UR                  S9nXS S 2U R
                  U R                  4'   XDR                  SS5      -
  nU$ )Nr   ©r   Údtyper   r   )r   r    r#   r   rV   r@   rB   Ú	transpose)r   ÚvecrE   Ú
batch_sizeÚmatrixs        r   Ú_pytorch_skew_symmetricÚ)OFTRotationModule._pytorch_skew_symmetrich   s]   € Ø—Y‘Y˜q‘\ˆ
Ü—’˜Z°ZÏ
É
ÐZ]×ZcÑZcÑdˆà*-Šq�$—)‘)˜TŸY™YÐ&Ñ'Ø×*Ñ*¨2¨rÓ2Ñ2ˆØˆr   c                ó`   • UR                   S   nUS S 2U R                  U R                  4   nU$ )Nr   )r   r@   rB   )r   rZ   rE   rY   rX   s        r   Ú_pytorch_skew_symmetric_invÚ-OFTRotationModule._pytorch_skew_symmetric_invp   s/   € Ø—\‘\ !‘_ˆ
ð ’Q˜Ÿ	™	 4§9¡9Ð,Ñ-ˆØˆ
r   c                ó\  • UR                   u  pVUR                  nU R                  X5      nU(       aâ  [        R                  " X!R
                  UR                  S9R                  USS5      n	US:”  a¡  U	R                  USS9  US:”  a‹  [        R                  " Xˆ5      n
U	R                  U
SS9  U
n[        SUS-
  5       H)  n[        R                  " X¸5      nU	R                  USS9  M+     [        R                  " X¸5      nU	R                  U5        Oˆ[        R                  " UR                   S   UR
                  S9R                  S	5      R                  XXR                   S   UR                   S   5      n[        R                  R                  XÈ-   XÈ-
  S
S9n	U	R                  U5      $ )z“
Perform the Cayley parametrization on a batch of skew-symmetric matrices.

Args:
    data: A batch of skew-symmetric matrices of shape (b, r, c).
rU   r	   g       @)Úalphaé   é   r   r   r   F)Úleft)r   rV   r[   r    r&   r   r'   Úadd_ÚbmmÚrangeÚ	unsqueezeÚexpandÚlinalgÚsolveÚto)r   ÚQrE   rO   Únum_neumann_termsÚbr+   Úprevious_dtypeÚQ_skewÚRÚ	Q_squaredÚQ_powerÚid_mats                r   Ú_cayley_batchÚOFTRotationModule._cayley_batchw   sc  € ð �w‰w‰ˆØŸ™ˆð ×-Ñ-¨aÓ<ˆæÜ—	’	˜*¯X©X¸Q¿W¹WÑE×LÑLÈQÐPQÐSTÓUˆAØ  1Ó$Ø—‘�v S�Ñ)Ø$ qÓ(Ü %§	¢	¨&Ó 9�IØ—F‘F˜9¨C�FÑ0à'�GÜ" 1Ð&7¸!Ñ&;Ö<˜Ü"'§)¢)¨GÓ"<˜ØŸ™˜w¨c˜Ó2ñ =ô $Ÿiši¨Ó8�GØ—F‘F˜7”Oøô —	’	˜&Ÿ,™, rÑ*°6·=±=ÑAß‘˜1“ß‘˜Ÿ<™<¨Ñ+¨V¯\©\¸"Ñ-=Ó>ð ô
 —‘×"Ñ" 6¡?°F±OÈ%Ð"ÐPˆAà�t‰t�NÓ#Ð#r   c                óP  • U R                  XR                  5      nUS-  [        R                  " [        R                  " UR
                  S   5      5      -  n[        R                  " UR                  S5      UR                  S5      4UR                  UR                  S9R                  S5      R                  U5      nX4-
  n[        R                  " X4-
  SSS9nXb:*  R                  5       n[        R                  " XsXBXV-  -  -   5      nU R                  X€R                  5      $ )Nr	   r   rU   )r	   rb   T)ÚdimÚkeepdim)r[   rE   r    ÚsqrtÚtensorr   r#   Úsizer   rV   rh   Ú	expand_asÚnormÚboolÚwherer^   )	r   rm   rL   Úoft_RÚIÚdiffÚ	norm_diffr.   Úouts	            r   Ú_project_batchÚ OFTRotationModule._project_batch    sä   € Ø×,Ñ,¨Q·±Ó@ˆà�A‰gœŸ
š
¤5§<¢<°·±¸A±Ó#?Ó@Ñ@ˆä�KŠK˜Ÿ™ A›¨¯
©
°1«Ð6¸u¿|¹|ÐSX×S^ÑS^Ñ_ß‰Y�q‹\ß‰Y�uÓð 	
ð
 ‰yˆÜ—J’J˜u™y¨f¸dÑCˆ	ØÑ ×&Ñ&Ó(ˆÜ�kŠk˜$ q°$Ñ2BÑ+CÑ'CÓDˆà×/Ñ/°·_±_ÓEÐEr   c                óà   • UR                   S   S:X  a  [        U5       Vs/ s H  o1S   PM	     nnO[        U5       Vs/ s H	  o1US4   PM     nn[        R                  " U6 nU$ s  snf s  snf )Nr   r	   )r   ..)r   rg   r    Ú
block_diag)r   r‚   ÚrankÚiÚblocksÚAs         r   Ú_block_diagonalÚ!OFTRotationModule._block_diagonal±   sk   € Ø�;‰;�q‰>˜QÓä-2°4¬[Ó9ª[¨˜F”m©[ˆFÐ9ˆFä-2°4¬[Ó9ª[¨˜A˜s˜F”m©[ˆFÐ9ô ×Ò˜fÐ%ˆàˆùò :ùâ9s
   ¡A&¿A+c                ó¸  • UR                   u  p#pE[        U R                  [        5      (       a  U R                  U R                  pvOU R                  u  pgS=p‰S=p«USU
-  -   U-
  U-  S-   nUSU-  -   U-
  U	-  S-   nUR	                  SXh5      R	                  SXy5      nUR                  SSSSSS5      R                  5       nUR                  X,-  U-  S5      nU$ )z|
Unfold with stride=1, padding=0 to preserve spatial dimensions. Only use kernel_size from base layer to define
patch size.
r	   r   rb   rc   é   é   r   )r   Ú
isinstancerN   r   ÚunfoldÚpermuteÚ
contiguousr%   )r   r(   rY   Úin_channelsÚ	in_heightÚin_widthÚkernel_heightÚkernel_widthÚstride_hÚstride_wÚpad_hÚpad_wÚ
out_heightÚ	out_widthÚ
x_unfoldeds                  r   Ú_unfoldÚOFTRotationModule._unfold½   sò   € ð
 89·w±wÑ4ˆ
 ä�d×&Ñ&¬×,Ñ,Ø*.×*:Ñ*:¸D×<LÑ<L™<à*.×*:Ñ*:Ñ'ˆMàÐˆØÐˆð   ! e¡)Ñ+¨mÑ;ÀÑHÈ1ÑLˆ
Ø  E¡	Ñ)¨LÑ8¸XÑEÈÑIˆ	ð —X‘X˜a Ó9×@Ñ@ÀÀLÓ[ˆ
Ø×'Ñ'¨¨1¨a°°A°qÓ9×DÑDÓFˆ
Ø—_‘_ ZÑ%<¸yÑ%HÈ"ÓMˆ
àÐr   c                ó†  • Uu  p4pV[        U R                  [        5      (       a  U R                  U R                  p‡OU R                  u  pxXW-
  S-   n	Xh-
  S-   n
UR                  X9X¤Xx5      nUR	                  SSSSSS5      R                  5       n[        R                  " UR                  X4U-  U-  Xš-  5      XV4Xx4SS9nU$ )	z+
Fold back to preserve spatial dimensions.
r	   r   rc   rb   r’   r“   )r	   r	   )Úoutput_sizerN   Ústride)r”   rN   r   r%   r–   r—   ÚFÚfold)r   r£   Ú
orig_shaperY   r˜   r™   rš   r›   rœ   r¡   r¢   Ú
x_reshapedÚx_foldeds                r   Ú_foldÚOFTRotationModule._fold×   sÛ   € ð 8BÑ4ˆ
 ä�d×&Ñ&¬×,Ñ,Ø*.×*:Ñ*:¸D×<LÑ<L™<à*.×*:Ñ*:Ñ'ˆMð Ñ.°Ñ2ˆ
ØÑ+¨aÑ/ˆ	ð  —_‘_ Z¸YÐUbÓqˆ
ð  ×'Ñ'¨¨1¨a°°A°qÓ9×DÑDÓFˆ
ô —6’6Ø�O‰O˜J°mÑ(CÀlÑ(RÐT^ÑTjÓkØ"Ð-Ø&Ð5Øñ	
ˆð ˆr   c                ój  • UR                   nX R                  R                   :w  a%  UR                  U R                  R                   5      nUR                  nU R                  (       a[  [
        R                  " 5          U R                  R                  U R                  U R                  U R                  S95        S S S 5        U R                  U R                  U R                  U R                  U R                  5      n[        U5      S:X  a  U R                  U5      nUR                  nU R                   (       a  U R"                  U R                  -  OU R$                  nUR                  S S nUR&                  " / UQUPU R                  P76 nU R                   (       a+  UR)                  USS5      n[
        R*                  " SX„5      n	O[
        R*                  " SX„5      n	U	R&                  " U6 n
[        U5      S:X  a  U R-                  X£5      n
U
R                  U5      $ ! , (       d  f       GNo= f)N©rL   r’   r   r	   z...rk,rkc->...rc)rV   rJ   rl   r   rK   r    Úno_gradÚcopy_r‡   rL   rv   rE   rO   rP   Úlenr¤   rM   rF   rC   Úreshaper'   Úeinsumr®   )r   r(   Úrequired_dtyper«   Úorth_rotateÚfolded_shaper‹   Ú
batch_dimsr¬   Úx_rotated_reshapedÚ	x_rotateds              r   r0   ÚOFTRotationModule.forwardö   s¢  € ð
 Ÿ™ˆØŸ[™[×.Ñ.Ó.Ø—‘�T—[‘[×&Ñ&Ó'ˆAà—W‘Wˆ
à�9�9Ü—’•Ø—‘×!Ñ! $×"5Ñ"5°d·k±kÀtÇxÁxÐ"5Ð"PÔQ÷ !ð ×(Ñ(Ø�K‰K˜Ÿ™¨$×*AÑ*AÀ4×C`ÑC`ó
ˆô
 ˆz‹?˜aÓØ—‘˜Q“ˆAà—w‘wˆØ6:×6F×6Fˆt×Ñ 4§?¡?Ò2ÈDÏFÉFˆØ—W‘W˜S˜b�\ˆ
Ø—Y’YÐB 
ÐB¨DÐB°$·/±/ÒBˆ
à××Ø%×,Ñ,¨T°1°aÓ8ˆKÜ!&§¢Ð.@À*Ó!ZÑä!&§¢Ð.@À*Ó!ZÐà&×.Ò.°Ð=ˆ	äˆz‹?˜aÓØŸ
™
 9Ó9ˆIà�|‰|˜NÓ+Ð+÷7 !–ús   Á=>H#È#
H2c                óò  • U R                   nU R                  (       aR  [        R                  " 5          U R	                  XR
                  S9nU R                   R                  U5        SSS5        U R                  XR                  U R                  U R                  5      nU R                  (       d  U R                  OU R                  U R                  -  nU R                  X#5      $ ! , (       d  f       N†= f)úš
Compute the delta weight for the given adapter.

Args:
    adapter (str):
        The name of the adapter for which the delta weight should be computed.
r±   N)rJ   rK   r    r²   r‡   rL   r³   rv   rE   rO   rP   rM   rC   rF   r�   )r   rJ   r¸   r‹   s       r   Ú
get_weightÚOFTRotationModule.get_weight  s±   € ð —‘ˆà�9�9Ü—’•Ø×,Ñ,¨V¿¹Ð,ÐB�Ø—‘×!Ñ! &Ô)÷ !ð ×(Ñ(Ø—O‘O T×%<Ñ%<¸d×>[Ñ>[ó
ˆð "×-×-ˆt�vŠv°4×3CÑ3CÀtÇÁÑ3VˆØ×#Ñ# KÓ6Ð6÷ !•ús   ³5C(Ã(
C6)rM   rE   rK   rL   rF   rN   rD   rP   rC   rO   rJ   )FçiUMu?F)r   r   Tr“   )Tr“   )
rm   útorch.TensorrE   r   rO   r€   rn   r   ÚreturnrÃ   )gñhãˆµøä>)r‚   rÃ   r‹   r   rÄ   rÃ   )r4   r5   r6   r7   r   r[   r^   rv   r‡   r�   r¤   r®   r0   rÀ   r9   r:   r;   s   @r   r=   r=   H   s€   ø† ð ØØØØØ!"÷=ò>òð klð&$Øð&$Ø+.ð&$ØDHð&$Ødgð&$à	õ&$ôRFô"
òò4ò>',÷R7ð 7r   r=   c                  ó|   • \ rS rSr% SrSrS\S'   SrS\S'   SS jrS	 r	SS
 jr
SSS jjr S SS jjrS rS rSrg)ÚOFTLayeri6  z
Implements the OFT layer.
)r‚   Úoft_embedding_Rztuple[str, ...]Úadapter_layer_names)rC   Úoft_block_sizeÚoft_dropoutÚother_param_namesc                óJ  • Xl         [        R                  " 0 5      U l        [        R                  " 0 5      U l        0 U l        0 U l        0 U l        [        R                  " 0 5      U l        SU l        / U l	        SU l
        X l        U R                  5       n[        U[        R                  5      (       a  UR                  UR                   pCGOH[        U[        R"                  5      (       a  UR$                  UR&                  pCGO[        U[        R(                  5      (       a  UR*                  UR,                  pCGOØ[/        US5      (       a*  [/        US5      (       a  UR0                  UR2                  pCGO�[/        US5      (       a*  [/        US5      (       a  UR4                  UR6                  pCGOb[/        US5      (       a3  UR8                  R:                  S:X  a  UR                  UR                   pCGO[/        US	5      (       a2  UR8                  R:                  S
:X  a  UR                  UR                   pCOÛUR8                  R:                  S:X  a  UR                  UR                   pCO©[/        US5      (       a2  UR8                  R:                  S:X  a  UR                  UR                   pCOf[/        US5      (       a)  [/        US5      (       a  UR                  UR                   pCOSu  p4[<        R>                  " S[A        U5       S3[B        5        X0l        X@l        g)zÌ
Initializes the OFT layer.

Note, currently only support linear layer and convolutional layer, with further support for other layers to be
added soon.

Parameters:
base_layer: the pretrained model layer
FTÚ
infeaturesÚoutfeaturesÚ
input_sizer§   Ú	codebooksÚQuantizedLinearÚbitsÚAwqGEMMQuantLinearÚ
EetqLinearÚW_qÚ	HQQLinearrF   Úout_features)NNzUnsupported layer type 'z(' encountered, proceed at your own risk.N)"Ú
base_layerrG   Ú
ModuleDictr‚   rÇ   rÉ   rC   rÊ   Ú_disable_adaptersÚmerged_adaptersÚcast_input_dtype_enabledÚkwargsÚget_base_layerr”   ÚLinearrF   r×   ÚConv2dr˜   Úout_channelsÚ	EmbeddingÚembedding_dimÚnum_embeddingsÚhasattrrÍ   rÎ   rÏ   r§   r   r4   ÚwarningsÚwarnÚtypeÚUserWarning)r   rØ   rÝ   rF   r×   s        r   r   ÚOFTLayer.__init__@  si  € ð %ŒÜ—]’] 2Ó&ˆŒ
ä!Ÿ}š}¨RÓ0ˆÔØ ˆÔØˆŒØ ˆÔÜŸ=š=¨Ó,ˆÔà!&ˆÔØ!ˆÔà(,ˆÔ%ØŒà×(Ñ(Ó*ˆ
Ü�j¤"§)¡)×,Ñ,Ø(2×(>Ñ(>À
×@WÑ@WšÜ˜
¤B§I¡I×.Ñ.Ø(2×(>Ñ(>À
×@WÑ@WšÜ˜
¤B§L¡L×1Ñ1Ø(2×(@Ñ(@À*×B[ÑB[šÜ�Z ×.Ñ.´7¸:À}×3UÑ3Uà(2×(=Ñ(=¸z×?UÑ?UšÜ�Z ×.Ñ.´7¸:À}×3UÑ3Uà(2×(=Ñ(=¸z×?UÑ?UšÜ�Z ×-Ñ-°*×2FÑ2F×2OÑ2OÐSdÓ2dà(2×(>Ñ(>À
×@WÑ@WšÜ�Z ×(Ñ(¨Z×-AÑ-A×-JÑ-JÐNbÓ-bà(2×(>Ñ(>À
×@WÑ@W™Ø×!Ñ!×*Ñ*¨lÓ:à(2×(>Ñ(>À
×@WÑ@W™Ü�Z ×'Ñ'¨J×,@Ñ,@×,IÑ,IÈ[Ó,Xà(2×(>Ñ(>À
×@WÑ@W™ô �z =×1Ñ1´g¸jÈ.×6YÑ6YØ,6×,BÑ,BÀJ×D[ÑD[™\à,6Ñ)�Ü�MŠMØ*¬4°
Ó+;Ð*<Ð<dÐeÔgrôð 'ÔØ(Õr   c                óP   • XR                   ;  a  g [        R                  " S5        g )NúFScaling operation for OFT not supported! Automatically set scale to 1.)Úscalingræ   rç   )r   ÚadapterÚscales      r   Ú	set_scaleÚOFTLayer.set_scale  s   € ØŸ,™,Ó&àä�ŠÐ^Õ_r   c                ó¢   • US:X  a  g U R                    H8  nX R                  R                  5       ;  a  M"  [        R                  " S5        M:     g )Nr	   rì   ©Úactive_adaptersr‚   Úkeysræ   rç   ©r   rï   Úactive_adapters      r   Úscale_layerÚOFTLayer.scale_layer†  s=   € Ø�A‹:Øà"×2Ô2ˆNØ§Z¡Z§_¡_Ó%6Ó6Ùä�MŠMÐbÖcò	 3r   Nc                ó”   • U R                    H8  nX R                  R                  5       ;  a  M"  [        R                  " S5        M:     g )Nz>Unscaling operation for OFT not supported! Keeping scale to 1.ró   rö   s      r   Úunscale_layerÚOFTLayer.unscale_layer�  s3   € Ø"×2Ô2ˆNØ§Z¡Z§_¡_Ó%6Ó6Ùä�MŠMÐZÖ[ò	 3r   c                óô  •  US:”  a
  [        US9nO[        R                  " 5       nU R                  R	                  [        R
                  " X05      5        US:X  a|  US:w  av  U R                  U-  S:w  d  X0R                  :”  a;  UnU R                  U R                  U5      n[        R                  " SU SU S35        [        U R                  U-  5      nO�US:w  a|  US:X  av  U R                  U-  S:w  d  X R                  :”  a;  UnU R                  U R                  U5      n[        R                  " SU SU S35        [        U R                  U-  5      nO[        S	5      eX3S
-
  -  S-  n[        U(       d  UOS
UUU R                  UUUU	U
S9	U R                  U'   U R                  X5        X R                  U'   X0R                   U'   U R#                  U5        U R%                  U R&                  US9  g)zU
Update the linear layer with trainable OFT weights. Override for other layer types.
r3   r2   r   úInvalid `oft_block_size` (ú!)! Adjusted `oft_block_size` to (ú).úInvalid `r` (ú)! Adjusted `r` to (úZSomething went wrong, please report this error: https://github.com/huggingface/peft/issuesr	   rb   ©rK   rL   rM   rO   rP   ©Úinference_modeN)r   rG   ÚIdentityrÊ   ÚupdaterÙ   rF   Úadjust_oft_parametersræ   rç   r   r   r=   r‚   Úreset_oft_parametersrC   rÉ   Ú%_move_adapter_to_device_of_base_layerÚset_adapterrô   )r   Úadapter_namerC   rÉ   Úmodule_dropoutrK   rL   rM   Úinit_weightsrO   rP   r  rÝ   Úoft_dropout_layerÚold_oft_block_sizeÚold_rrD   s                    r   Úupdate_layerÚOFTLayer.update_layer—  sø  € ð$	ð ˜CÓÜ :¸^Ñ LÑä "§¢£ÐØ×Ñ×Ñ¤§¢¨|Ð.OÓ PÔQà�‹6�n¨Ó)Ø×Ñ .Ñ0°AÓ5¸×JZÑJZÓ9ZØ%3Ð"Ø!%×!;Ñ!;¸D×<LÑ<LÈnÓ!]�Ü—’Ø0Ð1CÐ0DÐDeÐftÐeuÐuwÐxôô �D×$Ñ$¨Ñ6Ó7‰AØ�!‹V˜¨!Ó+Ø×Ñ !Ñ# qÓ(¨A×0@Ñ0@Ó,@Ø�Ø×.Ñ.¨t×/?Ñ/?ÀÓC�Ü—’ ¨e¨WÐ4HÈÈÈ2ÐNÔOÜ  ×!1Ñ!1°QÑ!6Ó7‰NäØlóð ð
 $¸Ñ'9Ñ:¸aÑ?ˆ
Ü#4Þ ‰A aØØØ×ÑØØØ#Ø1Ø%=ñ
$
ˆ�
‰
�<Ñ ð 	×!Ñ! ,Ô=ð  !�‰ˆ|ÑØ,:×Ñ˜LÑ)ð 	×2Ñ2°<Ô@Ø×Ñ˜×-Ñ-¸nÐÒMr   c                óþ  • USL a¨  XR                   R                  5       ;   a7  [        R                  R	                  U R                   U   R
                  SSS9  gXR                  R                  5       ;   a7  [        R                  R	                  U R                  U   R
                  SSS9  gXR                   R                  5       ;   aK  USL a7  [        R                  R                  U R                   U   R
                  5        O[        SU< 35      eXR                  R                  5       ;   aK  USL a7  [        R                  R                  U R                  U   R
                  5        g[        SU< 35      eg)z
Reset the OFT parameters.
Fr3   gš™™™™™¹?)ÚmeanÚstdNTz$Unknown initialization init_weights=)	r‚   rõ   rG   ÚinitÚnormal_rJ   rÇ   Úzeros_r   )r   r  r  s      r   r
  ÚOFTLayer.reset_oft_parametersê  s'  € ð ˜5Ò ØŸz™zŸ™Ó0Ó0Ü—‘—‘ §
¡
¨<Ñ 8× ?Ñ ?ÀcÈs�ÑSØØ×3Ñ3×8Ñ8Ó:Ó:Ü—‘—‘ × 4Ñ 4°\Ñ B× IÑ IÐPSÐY\�Ñ]ØàŸ:™:Ÿ?™?Ó,Ó,Ø˜tÒ#ä—‘—‘˜tŸz™z¨,Ñ7×>Ñ>Õ?ä Ð#H¸<¹/Ð!JÓKÐKØ×/Ñ/×4Ñ4Ó6Ó6Ø˜tÒ#ä—‘—‘˜t×3Ñ3°LÑA×HÑHÕIä Ð#H¸<¹/Ð!JÓKÐKð 7r   c                ó¾   • X!:  a$  UnX1::  a  X-  S:w  a  US-  nX1::  a
  X-  S:w  a  M  OU$ UnUS:”  a  X-  S:w  a  US-  nUS:”  a
  X-  S:w  a  M  X$-
  X2-
  ::  a  U$ U$ )zI
Adjust the OFT parameters to be divisible by the in_features dimension.
r   r	   © )r   rF   ÚparamsÚhigher_paramsÚlower_paramss        r   r	  ÚOFTLayer.adjust_oft_parameters  s—   € ð ÓØ"ˆMØÓ.°;Ñ3NÐRSÓ3SØ Ñ"�ð  Ó.°;Ñ3NÐRSÕ3Søð ÐàˆØ˜QÓ ;Ñ#=ÀÓ#BØ˜AÑˆLð ˜QÓ ;Ñ#=ÀÕ#Bð Ñ! }Ñ'=Ó>ØÐà Ð r   )rÚ   rØ   rÜ   rF   rÝ   rÛ   r‚   rÉ   rÊ   rÇ   r×   rC   )rØ   ú	nn.ModulerÄ   ÚNone)rï   ÚfloatrÄ   r#  ©N©rÄ   r#  ©F©r  r€   )r4   r5   r6   r7   r8   rÈ   Ú__annotations__rË   r   rð   rø   rû   r  r
  r	  r9   r  r   r   rÆ   rÆ   6  s\   ‡ ñð
 ,HÐ˜ÓGà)OÐ�ÓOô=)ò~`ôdö\ð&  %ðQNð õQNòfLõ2!r   rÆ   c                  ó¸   ^ • \ rS rSrSr           S
                         SU 4S jjjrSSS jjrSS jrSS jrSS jr	SU 4S jjr
S	rU =r$ )rß   i  zOFT implemented in Linear layerc                ó¤   >• [         TU ]  5         [        R                  " X40 UD6  X°l        X l        U R                  UUUUUUUUU	U
S9
  XÐl        g ©N)rÉ   r  rK   rL   rM   r  rO   rP   )r   r   rÆ   Úfan_in_fan_outÚ_active_adapterr  Úis_target_conv_1d_layer)r   rØ   r  rC   rÉ   r  rK   rL   rM   rO   rP   r-  r  r/  rÝ   r   s                  €r   r   ÚLinear.__init__  sh   ø€ ô" 	‰ÑÔÜ×Ò˜$Ñ5¨fÒ5Ø,Ôà+Ôà×ÑØØØ)Ø)ØØØ#Ø%Ø1Ø%=ð 	ñ 	
ð (?Õ$r   c                óP  • [        X5      nU(       d  gU GH  nX0R                  R                  5       ;   d  M#  U R                  5       nUR                  R
                  nU(       aí  UR                  R                  nU R                  U5      n[        R                  " USS5      n[        R                  " XvR                  UR
                  5      5      n[        R                  " USS5      n[        R                  " U5      R                  5       (       d  [        SU S35      eUR                  5       R                  U5      UR                  l        O´UR                  R                  nU R                  U5      n[        R                  " USS5      n[        R                  " XvR                  UR
                  5      5      n[        R                  " USS5      nUR                  5       R                  U5      UR                  l        U R                   R#                  U5        GM     g)a  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If `True`, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`List[str]`, *optional*):
        The list of adapter names that should be merged. If `None`, all active adapters will be merged.
        Defaults to `None`.
Nr   r	   ú1NaNs detected in the merged weights. The adapter ú seems to be broken)r   r‚   rõ   rÞ   rJ   rV   ÚdataÚget_delta_weightr    rW   Úmmrl   ÚisfiniteÚallr   r—   rÛ   Úappend©r   Ú
safe_mergeÚadapter_namesr÷   rØ   Ú
orig_dtypeÚorig_weightsÚoft_mats           r   ÚmergeÚLinear.merge@  sŸ  € ô 0°ÓDˆÞàä+ˆNØ§¡§¡Ó!2Õ2Ø!×0Ñ0Ó2�
Ø'×.Ñ.×4Ñ4�
Þà#-×#4Ñ#4×#9Ñ#9�LØ"×3Ñ3°NÓC�GÜ#(§?¢?°<ÀÀAÓ#F�LÜ#(§8¢8¨G·_±_ÀWÇ]Á]Ó5SÓ#T�LÜ#(§?¢?°<ÀÀAÓ#F�Lä Ÿ>š>¨,Ó7×;Ñ;×=Ñ=Ü(ØOÐP^ÐO_Ð_rÐsóð ð .:×-DÑ-DÓ-F×-IÑ-IÈ*Ó-U�J×%Ñ%Õ*à#-×#4Ñ#4×#9Ñ#9�LØ"×3Ñ3°NÓC�GÜ#(§?¢?°<ÀÀAÓ#F�LÜ#(§8¢8¨G·_±_ÀWÇ]Á]Ó5SÓ#T�LÜ#(§?¢?°<ÀÀAÓ#F�Là-9×-DÑ-DÓ-F×-IÑ-IÈ*Ó-U�J×%Ñ%Ô*à×$Ñ$×+Ñ+¨N×;ò7 ,r   c                óª  • U R                   (       d  [        R                  " S5        gU R                  5       nUR                  R
                  n[        U R                  5      S:”  Gak  U R                  R                  5       nX0R                  R                  5       ;   Ga  U R                  U5      nUR
                  nU[        R                  :w  a  UR                  [        R                  5      nU R                  5       R                  R                  n[        R                   " USS5      n[        R"                  " [        R$                  R'                  U5      R                  U5      UR                  U5      5      n[        R                   " USS5      nUR                  U5      UR                  l        [        U R                  5      S:”  a  GMj  gg©úG
This method unmerges all merged adapter layers from the base weights.
ú Already unmerged. Nothing to do.Nr   r	   )Úmergedræ   rç   rÞ   rJ   rV   r´   rÛ   Úpopr‚   rõ   r5  r    Úfloat32rl   r4  rW   r6  rj   Úinv©r   rØ   r=  r÷   r?  rp   r>  s          r   ÚunmergeÚLinear.unmergeo  sM  € ð �{�{Ü�MŠMÐ<Ô=Øà×(Ñ(Ó*ˆ
Ø×&Ñ&×,Ñ,ˆ
Ü�$×&Ñ&Ó'¨!Ô+Ø!×1Ñ1×5Ñ5Ó7ˆNØ§¡§¡Ó!2Ô2Ø×/Ñ/°Ó?�à!(§¡�Ø!¤U§]¡]Ó2Ø%Ÿj™j¬¯©Ó7�Gà#×2Ñ2Ó4×;Ñ;×@Ñ@�Ü$Ÿš¨|¸QÀÓB�Ü$Ÿxšx¬¯©×(8Ñ(8¸Ó(A×(DÑ(DÀ^Ó(TÐVb×VeÑVeÐftÓVuÓv�Ü$Ÿš¨|¸QÀÓB�à)5¯©¸Ó)D�
×!Ñ!Ô&ô �$×&Ñ&Ó'¨!×+Ð+r   c                óT  • U R                   U   R                  R                  nU R                   U   R                  R                  nUR                  S:H  =(       a-    U[
        R                  :H  =(       d    U[
        R                  :H  nU R                   U   nU(       az  UR                  R                  nUR                  R                  R                  5       UR                  l        UR                  5       nXeR                  l        UR                  U5      $ UR                  5       $ ©r¿   Úcpu©r‚   rJ   r   rV   rè   r    Úfloat16Úbfloat16r4  r$  rÀ   rl   ©r   rî   r   rV   Úcast_to_fp32Úoft_R_moduleÚoriginal_weightr?  s           r   r5  ÚLinear.get_delta_weight‰  óä   € ð —‘˜GÑ$×+Ñ+×2Ñ2ˆØ—
‘
˜7Ñ#×*Ñ*×0Ñ0ˆð
 —{‘{ eÑ+×c°¼%¿-¹-Ñ1G×1bÈ5ÔTY×TbÑTbÑKbˆà—z‘z 'Ñ*ˆæà*×1Ñ1×6Ñ6ˆOØ'3×':Ñ':×'?Ñ'?×'EÑ'EÓ'GˆL×ÑÔ$Ø"×-Ñ-Ó/ˆGØ'6×ÑÔ$Ø—:‘:˜eÓ$Ð$à×*Ñ*Ó,Ð,r   c                óH  • UR                   nU R                  (       a8  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nO»U R                  (       a  U R                  " U/UQ70 UD6nO“U R
                   H^  nX`R                  R                  5       ;  a  M"  U R                  U   nU R                  XR                  R                   5      nU" U5      nM`     U R                  " UR                  U5      /UQ70 UD6nUR                  U5      nU$ r%  ©rV   Údisable_adaptersrF  rK  rØ   rô   r‚   rõ   Ú_cast_input_dtyperJ   rl   ©r   r(   ÚargsrÝ   rp   Úresultr÷   r‚   s           r   r0   ÚLinear.forward¥  sæ   € ØŸ™ˆà× × Ø�{�{Ø—‘”Ø—_’_ QÐ8¨Ò8°Ñ8‰FØ�[�[Ø—_’_ QÐ8¨Ò8°Ñ8‰Fà"&×"6Ô"6�Ø!¯©¯©Ó):Ó:ÙØŸ
™
 >Ñ2�à×*Ñ*¨1¯l©l×.@Ñ.@ÓA�Ù˜!“H’ñ #7ð —_’_ Q§T¡T¨.Ó%9ÐK¸DÒKÀFÑKˆFà—‘˜>Ó*ˆØˆr   c                ó*   >• [         TU ]  5       nSU-   $ ©Nzoft.©r   Ú__repr__©r   Úrepr   s     €r   rd  ÚLinear.__repr__¼  ó   ø€ Ü‰gÑÓ ˆØ˜‰|Ðr   )r.  r-  r/  )é   r   r3   FrÂ   FFr“   FTF)r  ÚstrrC   r   rÉ   r   r  r$  rK   r€   rL   r$  rM   r€   rO   r€   rP   r   r-  r€   r  úUnion[bool, str]r/  r€   rÄ   r#  ©FN©r;  r€   r<  zOptional[list[str]]rÄ   r#  r&  ©rÄ   rÃ   )r(   rÃ   rÄ   rÃ   ©rÄ   rj  )r4   r5   r6   r7   r8   r   r@  rK  r5  r0   rd  r9   r:   r;   s   @r   rß   rß     sÖ   ø† Ù)ð ØØ #ØØØ!Ø#(Ø()Ø$Ø)-Ø(-ð#?ð ð#?ð ð	#?ð
 ð#?ð ð#?ð ð#?ð ð#?ð ð#?ð !ð#?ð #&ð#?ð ð#?ð 'ð#?ð "&ð#?ð  
÷!#?ð #?öJ-<ô^Eô4-ô8÷.õ r   rß   c                  óÈ   ^ • \ rS rSrSr          S                         SU 4S jjjr S SS jjrSSS jjrSS jrSS jr	SS jr
SU 4S	 jjrS
rU =r$ )rà   iÁ  zOFT implemented in Conv2d layerc                ó”   >• [         TU ]  5         [        R                  X5        XPl        X l        U R                  UUUUUUU	U
UUS9
  g r,  ©r   r   rÆ   r-  r.  r  )r   rØ   r  rC   rÉ   r-  r  rK   rL   rM   r  rO   rP   rÝ   r   s                 €r   r   ÚConv2d.__init__Ä  s]   ø€ ô  	‰ÑÔÜ×Ñ˜$Ô+Ø,Ôà+Ôð 	×ÑØØØ)Ø)ØØØ#Ø%Ø1Ø%=ð 	ò 	
r   c                óü  • US:”  a
  [        US9nO[        R                  " 5       nU R                  R	                  [        R
                  " X05      5        U R                  5       nUR                  S   S:”  a  [        S5      eU R                  UR                  S   -  UR                  S   -  nUS:X  aQ  US:w  aK  Xó-  S:w  d  X?:”  a0  UnU R                  Xó5      n[        R                  " SU SU S35        [        Xó-  5      nObUS:w  aQ  US:X  aK  Xò-  S:w  d  X/:”  a0  UnU R                  Xò5      n[        R                  " S	U S
U S35        [        Xò-  5      nO[        S5      eX3S-
  -  S-  n[        U(       d  UOSUUUUUUUR                  U	U
S9
U R                   U'   U R#                  X5        X R$                  U'   X0R&                  U'   U R)                  U5        U R+                  U R,                  US9  g)z5
Update the conv2d layer with trainable OFT weights.
r3   r2   r   r	   z1Conv2d with dilation > 1 is not supported by OFT.rþ   rÿ   r   r  r  r  rb   )rK   rL   rM   rN   rO   rP   r  N)r   rG   r  rÊ   r  rÙ   rÞ   Údilationr   rF   rN   r	  ræ   rç   r   r=   r‚   r
  rC   rÉ   r  r  rô   )r   r  rC   rÉ   r  rK   rL   rM   r  rO   rP   r  rÝ   r  rØ   Úconv_filter_dimr  r  rD   s                      r   r  ÚConv2d.update_layerè  s  € ð& ˜CÓÜ :¸^Ñ LÑä "§¢£ÐØ×Ñ×Ñ¤§¢¨|Ð.OÓ PÔQð ×(Ñ(Ó*ˆ
Ø×Ñ˜qÑ! AÓ%ÜÐPÓQÐQà×*Ñ*¨Z×-CÑ-CÀAÑ-FÑFÈ×I_ÑI_Ð`aÑIbÑbˆà�‹6�n¨Ó)ØÑ/°1Ó4¸Ó8XØ%3Ð"Ø!%×!;Ñ!;¸OÓ!\�Ü—’Ø0Ð1CÐ0DÐDeÐftÐeuÐuwÐxôô �OÑ5Ó6‰AØ�!‹V˜¨!Ó+ØÑ" aÓ'¨1Ó+>Ø�Ø×.Ñ.¨ÓB�Ü—’ ¨e¨WÐ4HÈÈÈ2ÐNÔOÜ  Ñ!5Ó6‰NäØlóð ð
 $¸Ñ'9Ñ:¸aÑ?ˆ
Ü#4Þ ‰A aØØØØØØ#Ø"×.Ñ.Ø1Ø%=ñ$
ˆ�
‰
�<Ñ ð 	×!Ñ! ,Ô=ð  !�‰ˆ|ÑØ,:×Ñ˜LÑ)ð 	×2Ñ2°<Ô@Ø×Ñ˜×-Ñ-¸nÐÒMr   c                ó<  • [        X5      nU(       d  gU GH  nX0R                  R                  5       ;   d  M#  U R                  5       nUR                  R
                  nU(       GaL  UR                  R                  R                  5       nU R                  U5      nUR                  U R                  U R                  UR                  S   -  UR                  S   -  5      n[        R                  " USS5      n[        R                  " XvR!                  UR
                  5      5      n[        R                  " USS5      nUR                  U R                  U R                  UR                  S   UR                  S   5      nUR#                  5       R!                  U5      UR                  l        GOJU R                  U5      nUR                  R                  R                  5       nUR                  U R                  U R                  UR                  S   -  UR                  S   -  5      n[        R                  " USS5      n[        R                  " XvR!                  UR
                  5      5      n[        R                  " USS5      nUR                  U R                  U R                  UR                  S   UR                  S   5      nUR#                  5       R!                  U5      UR                  l        U R$                  R'                  U5        GM     g)a  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`List[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
        to `None`.
Nr   r	   )r   r‚   rõ   rÞ   rJ   rV   r4  Úcloner5  r%   r×   rF   rN   r    rW   r6  rl   r—   rÛ   r9  r:  s           r   r@  ÚConv2d.merge5  sŠ  € ô 0°ÓDˆÞàä+ˆNØ§¡§¡Ó!2Õ2Ø!×0Ñ0Ó2�
Ø'×.Ñ.×4Ñ4�
ßð $.×#4Ñ#4×#9Ñ#9×#?Ñ#?Ó#A�LØ"×3Ñ3°NÓC�Gà#/×#4Ñ#4Ø×)Ñ)¨4×+;Ñ+;¸j×>TÑ>TÐUVÑ>WÑ+WÐZd×ZpÑZpÐqrÑZsÑ+só$�Lô $)§?¢?°<ÀÀAÓ#F�LÜ#(§8¢8¨G·_±_ÀWÇ]Á]Ó5SÓ#T�LÜ#(§?¢?°<ÀÀAÓ#F�LØ#/×#4Ñ#4Ø×)Ñ)¨4×+;Ñ+;¸Z×=SÑ=SÐTUÑ=VÐXb×XnÑXnÐopÑXqó$�Lð .:×-DÑ-DÓ-F×-IÑ-IÈ*Ó-U�J×%Ñ%Ö*à"×3Ñ3°NÓC�Gà#-×#4Ñ#4×#9Ñ#9×#?Ñ#?Ó#A�LØ#/×#4Ñ#4Ø×)Ñ)¨4×+;Ñ+;¸j×>TÑ>TÐUVÑ>WÑ+WÐZd×ZpÑZpÐqrÑZsÑ+só$�Lô $)§?¢?°<ÀÀAÓ#F�LÜ#(§8¢8¨G·_±_ÀWÇ]Á]Ó5SÓ#T�LÜ#(§?¢?°<ÀÀAÓ#F�LØ#/×#4Ñ#4Ø×)Ñ)¨4×+;Ñ+;¸Z×=SÑ=SÐTUÑ=VÐXb×XnÑXnÐopÑXqó$�Lð .:×-DÑ-DÓ-F×-IÑ-IÈ*Ó-U�J×%Ñ%Ô*à×$Ñ$×+Ñ+¨N×;òK ,r   c                óF  • U R                   (       d  [        R                  " S5        gU R                  5       nUR                  R
                  n[        U R                  5      S:”  Ga9  U R                  R                  5       nX0R                  R                  5       ;   Gaä  U R                  U5      nUR
                  nU[        R                  :w  a  UR                  [        R                  5      nU R                  5       R                  R                  R!                  5       nUR#                  U R$                  U R&                  U R                  5       R(                  S   -  U R                  5       R(                  S   -  5      n[        R*                  " USS5      n[        R,                  " [        R.                  R1                  U5      R                  U5      UR                  U5      5      n[        R*                  " USS5      nUR#                  U R$                  U R&                  U R                  5       R(                  S   U R                  5       R(                  S   5      nUR                  U5      UR                  l        [        U R                  5      S:”  a  GM8  ggrC  )rF  ræ   rç   rÞ   rJ   rV   r´   rÛ   rG  r‚   rõ   r5  r    rH  rl   r4  ry  r%   r×   rF   rN   rW   r6  rj   rI  rJ  s          r   rK  ÚConv2d.unmergen  sÿ  € ð �{�{Ü�MŠMÐ<Ô=Øà×(Ñ(Ó*ˆ
Ø×&Ñ&×,Ñ,ˆ
Ü�$×&Ñ&Ó'¨!Ô+Ø!×1Ñ1×5Ñ5Ó7ˆNØ§¡§¡Ó!2Ô2Ø×/Ñ/°Ó?�à!(§¡�Ø!¤U§]¡]Ó2Ø%Ÿj™j¬¯©Ó7�Gà#×2Ñ2Ó4×;Ñ;×@Ñ@×FÑFÓH�Ø+×0Ñ0Ø×%Ñ%Ø×$Ñ$ t×':Ñ':Ó'<×'HÑ'HÈÑ'KÑKÈd×NaÑNaÓNc×NoÑNoÐpqÑNrÑró �ô  %Ÿš¨|¸QÀÓB�Ü$Ÿxšx¬¯©×(8Ñ(8¸Ó(A×(DÑ(DÀ^Ó(TÐVb×VeÑVeÐftÓVuÓv�Ü$Ÿš¨|¸QÀÓB�Ø+×0Ñ0Ø×%Ñ%Ø×$Ñ$Ø×'Ñ'Ó)×5Ñ5°aÑ8Ø×'Ñ'Ó)×5Ñ5°aÑ8ó	 �ð *6¯©¸Ó)D�
×!Ñ!Ô&ô1 �$×&Ñ&Ó'¨!×+Ð+r   c                óT  • U R                   U   R                  R                  nU R                   U   R                  R                  nUR                  S:H  =(       a-    U[
        R                  :H  =(       d    U[
        R                  :H  nU R                   U   nU(       az  UR                  R                  nUR                  R                  R                  5       UR                  l        UR                  5       nXeR                  l        UR                  U5      $ UR                  5       $ rN  rP  rS  s           r   r5  ÚConv2d.get_delta_weight’  rX  r   c                óH  • UR                   nU R                  (       a8  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nO»U R                  (       a  U R                  " U/UQ70 UD6nO“U R
                   H^  nX`R                  R                  5       ;  a  M"  U R                  U   nU R                  XR                  R                   5      nU" U5      nM`     U R                  " UR                  U5      /UQ70 UD6nUR                  U5      nU$ r%  rZ  r]  s           r   r0   ÚConv2d.forward®  sæ   € ØŸ™ˆà× × Ø�{�{Ø—‘”Ø—_’_ QÐ8¨Ò8°Ñ8‰FØ�[�[Ø—_’_ QÐ8¨Ò8°Ñ8‰Fà"&×"6Ô"6�Ø!¯©¯©Ó):Ó:ÙàŸ
™
 >Ñ2�Ø×*Ñ*¨1¯l©l×.@Ñ.@ÓA�Ù˜!“H’ñ #7ð —_’_ Q§T¡T¨.Ó%9ÐK¸DÒKÀFÑKˆFà—‘˜>Ó*ˆØˆr   c                ó*   >• [         TU ]  5       nSU-   $ rb  rc  re  s     €r   rd  ÚConv2d.__repr__Å  rh  r   ©r.  r-  )
ri  r   Fr3   FrÂ   FTFr“   )rØ   r"  r  rj  rC   r   rÉ   r   r-  r€   r  r$  rK   r€   rL   r$  rM   r€   r  rk  rO   r€   rP   r   rÄ   r#  r'  r(  rl  rm  r&  rn  ©r(   rÃ   r^  r   rÝ   r   rÄ   rÃ   ro  )r4   r5   r6   r7   r8   r   r  r@  rK  r5  r0   rd  r9   r:   r;   s   @r   rà   rà   Á  së   ø† Ù)ð ØØ$Ø #ØØØ!Ø)-Ø#(Ø()ð"
àð"
ð ð"
ð ð	"
ð
 ð"
ð ð"
ð ð"
ð ð"
ð ð"
ð ð"
ð 'ð"
ð !ð"
ð #&ð"
ð 
÷"
ð "
ð`  %ðKNð õKNöZ7<ôr"EôH-ô8÷.õ r   rà   c                  óÎ   ^ • \ rS rSr          S                         SU 4S jjjr S SS jjrSSS jjrSS jrSS jrSS jr	SS jr
SU 4S	 jjrS
rU =r$ )râ   iÊ  c                ó”   >• [         TU ]  5         [        R                  X5        X°l        X l        U R                  UUUUUUUUU	U
S9
  g r,  rr  )r   rØ   r  rC   rÉ   r  rK   rL   rM   rO   rP   r-  r  rÝ   r   s                 €r   r   ÚEmbedding.__init__Ì  s[   ø€ ô  	‰ÑÔÜ×Ñ˜$Ô+Ø,Ôà+ÔØ×ÑØØØ)Ø)ØØØ#Ø%Ø1Ø%=ð 	ò 	
r   c                ó~  • [        5       R                  5       nUS	 US:X  a|  US:w  av  U R                  U-  S:w  d  X0R                  :”  a;  UnU R                  U R                  U5      n[        R
                  " SU SU S35        [        U R                  U-  5      nO�US:w  a|  US:X  av  U R                  U-  S:w  d  X R                  :”  a;  UnU R                  U R                  U5      n[        R
                  " SU SU S35        [        U R                  U-  5      nO[        S5      eX3S	-
  -  S
-  n[        U(       d  UOS	UUU R                  UUUU	U
S9	U R                  U'   U R                  X5        X R                  U'   X0R                  U'   U R                  U5        U R                  U R                  US9  g )Nr   r   rþ   rÿ   r   r  r  r  r	   rb   r  r  )ÚlocalsÚcopyrF   r	  ræ   rç   r   r   r=   rÇ   r
  rC   rÉ   r  r  rô   )r   r  rC   rÉ   r  rK   rL   rM   r  rO   rP   r  rÝ   r  r  rD   s                   r   r  ÚEmbedding.update_layerî  sÊ  € ô  “—‘“ˆØ�6ˆNà�‹6�n¨Ó)Ø×Ñ .Ñ0°AÓ5¸×JZÑJZÓ9ZØ%3Ð"Ø!%×!;Ñ!;¸D×<LÑ<LÈnÓ!]�Ü—’Ø0Ð1CÐ0DÐDeÐftÐeuÐuwÐxôô �D×$Ñ$¨Ñ6Ó7‰AØ�!‹V˜¨!Ó+Ø×Ñ !Ñ# qÓ(¨A×0@Ñ0@Ó,@Ø�Ø×.Ñ.¨t×/?Ñ/?ÀÓC�Ü—’ ¨e¨WÐ4HÈÈÈ2ÐNÔOÜ  ×!1Ñ!1°QÑ!6Ó7‰NäØlóð ð
 $¸Ñ'9Ñ:¸aÑ?ˆ
Ü->Þ ‰A aØØØ×ÑØØØ#Ø1Ø%=ñ
.
ˆ×Ñ˜\Ñ*ð 	×!Ñ! ,Ô=ð  !�‰ˆ|ÑØ,:×Ñ˜LÑ)ð 	×2Ñ2°<Ô@Ø×Ñ˜×-Ñ-¸nÐÒMr   c                ó”  • [        X5      nU(       d  gU GH®  nX0R                  R                  5       ;   d  M#  U R                  5       nUR                  R
                  nU(       a¾  UR                  R                  nU R                  U5      n[        R                  " UR                  UR
                  5      U5      n[        R                  " U5      R                  5       (       d  [        SU S35      eUR                  5       R                  U5      UR                  l        O…UR                  R                  nU R                  U5      n[        R                  " UR                  UR
                  5      U5      nUR                  5       R                  U5      UR                  l        U R                  R!                  U5        GM±     g)a  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`list[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
        to `None`.
Nr2  r3  )r   rÇ   rõ   rÞ   rJ   rV   r4  r5  r    r6  rl   r7  r8  r   r—   rÛ   r9  r:  s           r   r@  ÚEmbedding.merge-  s_  € ô 0°ÓDˆÞàä+ˆNØ×!5Ñ!5×!:Ñ!:Ó!<Õ<Ø!×0Ñ0Ó2�
Ø'×.Ñ.×4Ñ4�
Þà#-×#4Ñ#4×#9Ñ#9�LØ"×3Ñ3°NÓC�GÜ#(§8¢8¨L¯O©O¸G¿M¹MÓ,JÈGÓ#T�Lä Ÿ>š>¨,Ó7×;Ñ;×=Ñ=Ü(ØOÐP^ÐO_Ð_rÐsóð ð .:×-DÑ-DÓ-F×-IÑ-IÈ*Ó-U�J×%Ñ%Õ*à#-×#4Ñ#4×#9Ñ#9�LØ"×3Ñ3°NÓC�GÜ#(§8¢8¨L¯O©O¸G¿M¹MÓ,JÈGÓ#T�Là-9×-DÑ-DÓ-F×-IÑ-IÈ*Ó-U�J×%Ñ%Ô*Ø×$Ñ$×+Ñ+¨N×;ò- ,r   c                ó>  • U R                   (       d  [        R                  " S5        gU R                  5       nUR                  R
                  n[        U R                  5      S:”  Ga5  U R                  R                  5       nX0R                  R                  5       ;   aá  U R                  U5      nUR
                  nU[        R                  :w  a  UR                  [        R                  5      nU R                  5       R                  R                  n[        R                   " UR                  UR
                  5      [        R"                  R%                  U5      5      nUR                  U5      UR                  l        [        U R                  5      S:”  a  GM4  gg)rD  rE  Nr   )rF  ræ   rç   rÞ   rJ   rV   r´   rÛ   rG  rÇ   rõ   r5  r    rH  rl   r4  r6  rj   rI  rJ  s          r   rK  ÚEmbedding.unmergeW  s#  € ð �{�{Ü�MŠMÐ<Ô=Øà×(Ñ(Ó*ˆ
Ø×&Ñ&×,Ñ,ˆ
Ü�$×&Ñ&Ó'¨!Ô+Ø!×1Ñ1×5Ñ5Ó7ˆNØ×!5Ñ!5×!:Ñ!:Ó!<Ó<Ø×/Ñ/°Ó?�à!(§¡�Ø!¤U§]¡]Ó2Ø%Ÿj™j¬¯©Ó7�Gà#×2Ñ2Ó4×;Ñ;×@Ñ@�Ü$Ÿxšx¨¯©¸¿¹Ó(FÌÏÉ×HXÑHXÐY`ÓHaÓb�à)5¯©¸Ó)D�
×!Ñ!Ô&ô �$×&Ñ&Ó'¨!×+Ð+r   c                óT  • U R                   U   R                  R                  nU R                   U   R                  R                  nUR                  S:H  =(       a-    U[
        R                  :H  =(       d    U[
        R                  :H  nU R                   U   nU(       az  UR                  R                  nUR                  R                  R                  5       UR                  l        UR                  5       nXeR                  l        UR                  U5      $ UR                  5       $ rN  )rÇ   rJ   r   rV   rè   r    rQ  rR  r4  r$  rÀ   rl   rS  s           r   r5  ÚEmbedding.get_delta_weighto  sê   € ð ×%Ñ% gÑ.×5Ñ5×<Ñ<ˆØ×$Ñ$ WÑ-×4Ñ4×:Ñ:ˆð
 —{‘{ eÑ+×c°¼%¿-¹-Ñ1G×1bÈ5ÔTY×TbÑTbÑKbˆà×+Ñ+¨GÑ4ˆæà*×1Ñ1×6Ñ6ˆOØ'3×':Ñ':×'?Ñ'?×'EÑ'EÓ'GˆL×ÑÔ$Ø"×-Ñ-Ó/ˆGØ'6×ÑÔ$Ø—:‘:˜eÓ$Ð$à×*Ñ*Ó,Ð,r   c           
     óº   • U R                  5       n[        R                  " UUUR                  UR                  UR
                  UR                  UR                  S9$ )N)Úpadding_idxÚmax_normÚ	norm_typeÚscale_grad_by_freqÚsparse)rÞ   r©   Ú	embeddingr“  r”  r•  r–  r—  )r   ÚinputrJ   rØ   s       r   Ú_embedÚEmbedding._embed‹  sT   € Ø×(Ñ(Ó*ˆ
Ü�{Š{ØØØ"×.Ñ.Ø×(Ñ(Ø ×*Ñ*Ø)×<Ñ<Ø×$Ñ$ñ
ð 	
r   c                ó  • U R                   (       a7  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6$ U R                  (       a  U R                  " U/UQ70 UD6$ U R                  " U/UQ70 UD6nUR                  nU R
                   HP  nX`R                  ;  a  M  U R                  U   nU R                  XGR                  R                  5      nU" U5      nMR     UR                  U5      $ r%  )
r[  rF  rK  rØ   rV   rô   rÇ   r\  rJ   rl   )r   r(   r^  rÝ   r_  Ú	out_dtyper÷   rÇ   s           r   r0   ÚEmbedding.forward—  sÛ   € à× × Ø�{�{Ø—‘”Ø—?’? 1Ð6 tÒ6¨vÑ6Ð6Ø�;�;Ø—?’? 1Ð6 tÒ6¨vÑ6Ð6à—’ Ð4 TÒ4¨VÑ4ˆØ—L‘Lˆ	à"×2Ô2ˆNØ×%9Ñ%9Ó9ÙØ"×2Ñ2°>ÑBˆOØ×+Ñ+¨F×4JÑ4J×4PÑ4PÓQˆFÙ$ VÓ,ŠFñ 3ð �y‰y˜Ó#Ð#r   c                ó*   >• [         TU ]  5       nSU-   $ rb  rc  re  s     €r   rd  ÚEmbedding.__repr__¬  rh  r   rƒ  )
ri  r   r3   FrÂ   FFr“   FT)rØ   r"  r  rj  rC   r   rÉ   r   r  r$  rK   r€   rL   r$  rM   r€   rO   r€   rP   r   r-  r€   r  rk  rÄ   r#  r'  r(  rl  rm  r&  rn  )r™  rÃ   rJ   rÃ   rÄ   rÃ   r„  ro  )r4   r5   r6   r7   r   r  r@  rK  r5  rš  r0   rd  r9   r:   r;   s   @r   râ   râ   Ê  sê   ø† ð ØØ #ØØØ!Ø#(Ø()Ø$Ø)-ð 
àð 
ð ð 
ð ð	 
ð
 ð 
ð ð 
ð ð 
ð ð 
ð ð 
ð !ð 
ð #&ð 
ð ð 
ð 'ð 
ð 
÷ 
ð  
ð\  %ð=Nð õ=Nö~(<ôTEô0-ô8

ô$÷*õ r   râ   c                ó<  • S n[        U [        5      (       a  U R                  5       nOU n[        U[        R                  R
                  5      (       a  [        X40 UD6nU$ [        U[        R                  R                  5      (       a:  US   (       a"  [        R                  " S5        S=US'   Ul	        [        X40 UD6nU$ [        U[        R                  R                  5      (       a.  UR                  5       nUR                  SS 5        [        X40 UD6nU$ )Nr-  zjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.F)r”   r   rÞ   r    rG   rà   rß   ræ   rç   r-  râ   rŠ  rG  )Útargetr  Ú
oft_configrÝ   Ú
new_moduleÚtarget_base_layerÚembedding_kwargss          r   Údispatch_defaultr§  ±  s  € ð €Jä�&œ.×)Ñ)Ø"×1Ñ1Ó3Ñà"ÐäÐ#¤U§X¡X§_¡_×5Ñ5Ü˜FÑ;°FÑ;ˆ
ð Ðô 
Ð%¤u§x¡x§¡×	7Ñ	7ØÐ"×#Ü�MŠMð3ôð DIÐHˆFÐ#Ñ$ zÔ'@Ü˜FÑ;°FÑ;ˆ
ð Ðô 
Ð%¤u§x¡x×'9Ñ'9×	:Ñ	:Ø!Ÿ;™;›=ÐØ×ÑÐ-¨tÔ4Ü˜vÑHÐ7GÑHˆ
àÐr   )r¢  ztorch.nn.Moduler  rj  r£  r
   rÄ   zOptional[torch.nn.Module])Ú
__future__r   ræ   Útypingr   r   r   r    Útorch.nnrG   Útorch.nn.functionalÚ
functionalr©   Úpeft.tuners.tuners_utilsr   r   Úconfigr
   ÚModuler   r=   rÆ   rß   rà   râ   r§  r  r   r   Ú<module>r°     sÀ   ðõ #ã ß 'Ñ 'ã Ý ß Ð ç Lå ô) §¡ô )ôXk7˜Ÿ	™	ô k7ô\_!ˆ~ô _!ôDfˆR�Y‰Y˜ô fôRFˆR�Y‰Y˜ô FôRd�—	‘	˜8ô dðNØðàðð ðð
 õr   