ó
    >:j¼¼ ã                  óì  • S SK Jr  S SKrS SKrS SKrS SKJr  S SKJr  S SK	J
r
JrJr  S SKrS SKJr  S SKJr  S SKJs  Jr  S SKJr  S SKJr  S SKJr  S S	KJr  S S
KJrJrJ r   S SK!J"r"J#r#  S SK$J%r%J&r&J'r'J(r(  S SK)J*r*  S SK+J,r,  S SK-J.r.  SSK/J0r0  S/r1 " S S5      r2 " S S\5      r3 " S S\Rh                  \35      r5 " S S\Rh                  5      r6 " S S\Rh                  \35      r7 " S S\Rh                  \35      r8 " S S \85      r9 " S! S"\85      r: " S# S$\85      r; " S% S&\Rh                  \35      r< " S' S(\Rh                  5      r=S) r> " S* S+\Rh                  \35      r? S-         S.S, jjr@g)/é    )ÚannotationsN)ÚCallable)Úcontextmanager)ÚAnyÚOptionalÚUnion)Úsvd_lowrank)ÚConv1D)Úis_transformers_ge_v5_4_0)Ú
BufferDict)ÚBaseTunerLayerÚ_get_in_out_featuresÚcheck_adapters_to_merge)ÚALLOWED_COMPUTE_DTYPESÚUPCAST_DTYPES)Údequantize_module_weightÚgather_params_ctxÚget_bnb_param_typeÚskip_init_on_device)Ú
loftq_init)Ú	transpose)ÚPeftWarningé   )Ú
LoraConfigÚalora_offsetsc                  ó�   • \ rS rSrSr\S
S j5       r\SS j5       r\SS j5       r\SS j5       r	\          SS j5       r
Srg	)ÚLoraVarianté2   ac  
Base class for LoRA variants, e.g. DoRA.

This class should be subclassed and the methods below should be implemented accordingly. The methods should be
implemented as static methods, this makes it easier to combine variants.

Note for developers: These methods are prone to change and should thus considered to be "private". Use at your own
discretion.
c                ó   • [         e)zKInitialization code for the LoRA variant, it's called within `update_layer`©ÚNotImplementedError)ÚmoduleÚadapter_names     ÚS/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/lora/layer.pyÚinitÚLoraVariant.init=   ó
   € ô "Ð!ó    c                ó   • [         e)zZSafe merging of the weights from `merge(..., safe_merge=True)`, should return a new tensorr    ©r"   Úactive_adapterÚorig_weights      r$   Ú
merge_safeÚLoraVariant.merge_safeB   r'   r(   c                ó   • g)zdUnsafe merging of the weights from `merge(..., safe_merge=False)`, should modify the weight in-placeN© r*   s      r$   Úmerge_unsafeÚLoraVariant.merge_unsafeG   ó   � r(   c                ó   • g)zFRemove the adapter weights from the original weights, then return themNr0   r*   s      r$   ÚunmergeÚLoraVariant.unmergeK   r3   r(   c                ó   • [         e)a—  
The forward pass of the LoRA variant, should return the overall result (not just the diff)

Args:
    module (LoraLayer): The module on which the forward pass is called
    active_adapter (str): The name of the active adapter
    x (torch.Tensor): The input to the forward call
    result (torch.Tensor): The result from the base model
    **kwargs: Additional arguments passed from [`LoraLayer.forward`].
r    )r"   r+   ÚxÚresultÚkwargss        r$   ÚforwardÚLoraVariant.forwardO   s
   € ô$ "Ð!r(   r0   N)r"   Ú	LoraLayerr#   ÚstrÚreturnÚNone)r"   r=   r+   r>   r,   útorch.Tensorr?   rA   )r"   r=   r+   r>   r,   rA   r?   r@   )
r"   r=   r+   r>   r8   rA   r9   rA   r?   rA   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ústaticmethodr%   r-   r1   r5   r;   Ú__static_attributes__r0   r(   r$   r   r   2   s�   † ñð ó"ó ð"ð ó"ó ð"ð ósó ðsð óUó ðUð ð"Øð"àð"ð ð"ð ð	"ð 
ó"ó ó"r(   r   c                  ó"  • \ rS rSr% SrS\S'   SrS\S'   SSS jjrSS jrSS	 jr	          SS
 jr
S rS rS rS rS S jr\R"                  " 5       S 5       rS rS!S jrS"S jrS#S jrS$S jrS%S&S jjrS r          S'S jrSrg)(r=   éd   )Úlora_AÚlora_BÚlora_embedding_AÚlora_embedding_Bztuple[str, ...]Úadapter_layer_names)ÚrÚ
lora_alphaÚscalingÚlora_dropoutÚother_param_namesc                ót  • Xl         0 U l        0 U l        0 U l        [        R
                  " 0 5      U l        [        R
                  " 0 5      U l        [        R
                  " 0 5      U l        [        R                  " 0 5      U l
        [        R                  " 0 5      U l        SU l        / U l        0 U l        0 U l        0 U l        ["        R                  R                  5       U l        0 U l        X l        SU l        0 U l        X0l        U R1                  5       nU R3                  U5      u  pEX@l        XPl        g )NFT)Ú
base_layerrP   rQ   rR   ÚnnÚ
ModuleDictrS   rK   rL   ÚParameterDictrM   rN   Ú_disable_adaptersÚmerged_adaptersÚuse_doraÚ
use_rsloraÚ	lora_biasÚtorchÚlora_magnitude_vectorÚ_cachesÚephemeral_gpu_offloadÚcast_input_dtype_enabledÚlora_variantr:   Úget_base_layerr   Úin_featuresÚout_features)ÚselfrV   rb   r:   rf   rg   s         r$   Ú__init__ÚLoraLayer.__init__j   sü   € Ø$ŒØˆŒØˆŒØˆŒÜŸMšM¨"Ó-ˆÔÜ—m’m BÓ'ˆŒÜ—m’m BÓ'ˆŒä "× 0Ò 0°Ó 4ˆÔÜ "× 0Ò 0°Ó 4ˆÔà!&ˆÔØ!ˆÔØ)+ˆŒØ+-ˆŒØ*,ˆŒÜ%*§X¡X×%8Ñ%8Ó%:ˆÔ"Ø')ˆŒØ+@Ô"à.2ˆÔ%Ø46ˆÔØŒà×(Ñ(Ó*ˆ
Ø$(×$=Ñ$=¸jÓ$IÑ!ˆØ&ÔØ(Õr(   c                ó   • [        U5      $ ©N)r   )rh   r"   s     r$   r   ÚLoraLayer._get_in_out_featuresˆ   s   € Ü# FÓ+Ð+r(   c               ó   • g)a¾  Return a matching LoRA variant for this layer type.

Given the init arguments of this layer, return the correct LoRA variant, if any. E.g., if `use_dora=True`, this
method should return the DoRA variant for the given layer. If `use_alora=True`, same for aLoRA.

If there is no fitting variant, return None.

Note: If this layer type does not support the LoRA variant at all, please raise an error during __init__ as is
convention, and not here.

Nr0   )rh   Úconfigr:   s      r$   Úresolve_lora_variantÚLoraLayer.resolve_lora_variant‹   s   € ð r(   c                óž  • UR                   nUR                  nUR                  nUR                  n	UR                  n
UR                  SS5      nXµS'   UR                  SS 5      nUS::  a  [        SU 35      eU	(       a\  [        U R                  5       SS 5      c@  [        R                  " S[        U R                  5       5      R                   S3[        5        U R                  US	9nUb  XÐR                  U'   X R                   U'   X0R"                  U'   US
:”  a  [$        R&                  " US9nO[$        R(                  " 5       nU R                   R+                  [$        R,                  " X05      5        [$        R.                  " U R0                  USS9U R2                  U'   [$        R.                  " X R4                  U	S9U R6                  U'   U(       al  US   nUS   n[8        R$                  R;                  U5      U R2                  U   l        [8        R$                  R;                  U5      U R6                  U   l        X�R                  U'   U(       a'  U[>        R@                  " U5      -  U RB                  U'   OX2-  U RB                  U'   X€R                  U'   URD                  U RD                  U'   [G        U[H        5      (       aU  URK                  S5      (       a?  [M        U R                  5       R<                  5         U RO                  X5        S S S 5        GOý[G        U[H        5      (       aU  URK                  S5      (       a?  [M        U R                  5       R<                  5         U RQ                  X5        S S S 5        GO“[G        U[H        5      (       aS  URS                  5       S:X  a?  [M        U R                  5       R<                  5         U RU                  U5        S S S 5        GO+US:X  a>  [M        U R                  5       R<                  5         U RW                  X5        S S S 5        OçUS:X  a7  [$        RX                  R[                  U R6                  U   R<                  5        OªUS:X  a>  [M        U R                  5       R<                  5         U R]                  U5        S S S 5        OfUS:X  aH  [M        U R                  5       R<                  5         U R_                  XR`                  5        S S S 5        OU(       a  U Rc                  X5        U Re                  U5        XR                  ;   a"  U R                  U   RX                  " U 4XS.UD6  U Rg                  U Rh                  U
S9  [k        U S5      (       aY  U R                   HH  nUU Rl                  ;   d  M  U Rl                  U   Ro                  U R2                  U R6                  5        MJ     g g ! , (       d  f       NÕ= f! , (       d  f       Næ= f! , (       d  f       N÷= f! , (       d  f       GN	= f! , (       d  f       GN= f! , (       d  f       GN-= f)NÚtarget_nameÚ Útied_adapterr   ú?`r` should be a positive integer value but the value passed is Úbiasú;`lora_bias=True` was passed but the targeted layer of type úE has no bias. This means that merging LoRA weights won't be possible.©ro   ç        ©ÚpF©rw   rK   rL   ÚpissaÚcordaÚoloraÚloftqÚevaÚ
orthogonalÚlora_ga©r#   ro   ©Úinference_modeÚ
lora_arrow)8rS   Úinit_lora_weightsr]   r^   rˆ   ÚgetÚ
ValueErrorÚgetattrre   ÚwarningsÚwarnÚtyperB   r   rp   rd   rP   rQ   rW   ÚDropoutÚIdentityÚupdaterX   ÚLinearrf   rK   rg   rL   r_   Ú	ParameterÚweightÚmathÚsqrtrR   r\   Ú
isinstancer>   Ú
startswithr   Ú
pissa_initÚ
corda_initÚlowerÚ
olora_initr   r%   Úzeros_Úorthogonal_initÚlora_ga_initÚlora_ga_configÚreset_lora_parametersÚ%_move_adapter_to_device_of_base_layerÚset_adapterÚactive_adaptersÚhasattrr‰   Úon_adapter_change)rh   r#   rP   rQ   ro   r:   rS   rŠ   r]   r^   rˆ   rs   ru   rd   Úlora_dropout_layerÚlora_A_paramsÚlora_B_paramsÚadapters                     r$   Úupdate_layerÚLoraLayer.update_layer™   sô  € ð ×*Ñ*ˆØ"×4Ñ4ÐØ×&Ñ&ˆ
Ø×$Ñ$ˆ	Ø×.Ñ.ˆà—j‘j °Ó3ˆØ +ˆ}ÑØ—z‘z .°$Ó7ˆð �‹6ÜÐ^Ð_`Ð^aÐbÓcÐcæœ' $×"5Ñ"5Ó"7¸ÀÓFÑNÜ�MŠMØMÌdÐSW×SfÑSfÓShÓNi×NrÑNrÐMsð tWð Wäôð ×0Ñ0¸Ð0Ð?ˆØÑ#Ø.:×Ñ˜lÑ+à �‰ˆ|ÑØ(2�‰˜Ñ%Ø˜#ÓÜ!#§¢¨lÑ!;Ñä!#§¢£Ðà×Ñ× Ñ ¤§¢°Ð/QÓ!RÔSô %'§I¢I¨d×.>Ñ.>ÀÈÑ$Nˆ�‰�LÑ!Ü$&§I¢I¨a×1BÑ1BÈÑ$Sˆ�‰�LÑ!ö
 Ø(¨Ñ2ˆMØ(¨Ñ2ˆMä/4¯x©x×/AÑ/AÀ-Ó/PˆD�K‰K˜Ñ%Ô,Ü/4¯x©x×/AÑ/AÀ-Ó/PˆD�K‰K˜Ñ%Ô,à'0�‰�|Ñ$æØ)3´d·i²iÀ³lÑ)BˆD�L‰L˜Ò&à)3©ˆD�L‰L˜Ñ&à(2�‰˜Ñ%à&,§o¡oˆ�‰�lÑ#ô Ð'¬×-Ñ-Ð2C×2NÑ2NÈw×2WÑ2WÜ" 4×#6Ñ#6Ó#8×#?Ñ#?Õ@Ø—‘ Ô@÷ AÑ@äÐ)¬3×/Ñ/Ð4E×4PÑ4PÐQX×4YÑ4YÜ" 4×#6Ñ#6Ó#8×#?Ñ#?Õ@Ø—‘ Ô@÷ AÑ@äÐ)¬3×/Ñ/Ð4E×4KÑ4KÓ4MÐQXÓ4XÜ" 4×#6Ñ#6Ó#8×#?Ñ#?Õ@Ø—‘ Ô-÷ AÑ@à 'Ó)Ü" 4×#6Ñ#6Ó#8×#?Ñ#?Õ@Ø—‘ Ô5÷ AÐ@à %Ó'Ü�G‰G�N‰N˜4Ÿ;™; |Ñ4×;Ñ;Õ<Ø ,Ó.Ü" 4×#6Ñ#6Ó#8×#?Ñ#?Õ@Ø×$Ñ$ \Ô2÷ AÐ@à )Ó+Ü" 4×#6Ñ#6Ó#8×#?Ñ#?Õ@Ø×!Ñ! ,×0EÑ0EÔF÷ AÐ@æØ×&Ñ& |ÔGà×2Ñ2°<Ô@à×,Ñ,Ó,Ø×Ñ˜lÑ+×0Ò0°ÐjÀLÑjÐciÒjà×Ñ˜×-Ñ-¸nÐÑMô �4˜×&Ñ&Ø×,Ô,�Ø˜dŸo™oÕ-Ø—O‘O GÑ,×>Ñ>¸t¿{¹{ÈDÏKÉKÖXò -ð '÷G AÕ@ú÷ AÕ@ú÷ AÕ@ú÷ AÖ@ú÷
 AÖ@ú÷ AÖ@úsH   ÌW&Í0W7ÏXÐXÒX+Ó"X=×&
W4×7
XØ
XØ
X(Ø+
X:Ø=
Yc                ó²  • USLGa  XR                   R                  5       ;   Ga8  USL aJ  [        R                  R	                  U R                   U   R
                  [        R                  " S5      S9  OiUR                  5       S:X  aF  [        R                  R                  U R                   U   R
                  SU R                  U   -  S9  O[        SU< 35      e[        R                  R                  U R                  U   R
                  5        U R                  U   (       a6  [        R                  R                  U R                  U   R                  5        XR                   R                  5       ;   a¢  [        R                  R                  U R                   U   5        [        R                  R                  U R"                  U   5        U R                  U   (       a6  [        R                  R                  U R"                  U   R                  5        [$        R&                  " 5       (       Ga¡  [$        R(                  " 5       (       Ga…  U R+                  5       n[-        US	S 5      n[-        US
S 5      nUGbV  US;  a  [.        R0                  " SU S35        UR3                  5       n[$        R4                  " US5      nUS:X  aW  XR                   ;   aH  U R7                  U5        [$        R8                  " U R                   U   R
                  R:                  XvS9  g US:X  aW  XR                  ;   aH  U R7                  U5        [$        R8                  " U R                  U   R
                  R:                  XvS9  g US:X  aN  XR"                  ;   a>  U R7                  U5        [$        R8                  " U R"                  U   R:                  XvS9  g g g g g g )NFTé   )ÚaÚgaussianr   )Ústdz)Unknown initialization init_lora_weights=Ú_hf_tp_planÚ_hf_device_mesh)ÚcolwiseÚrowwiseÚembedding_rowwisez	tp_plan "z¹" found on the base layer is not supported for LoRA weight synchronization. Expected one of "colwise", "rowwise", "embedding_rowwise". LoRA weights may not be synchronized across ranks.r   r¶   )ÚsrcÚgroupr·   r¸   )rK   ÚkeysrW   r%   Úkaiming_uniform_r–   r—   r˜   r�   Únormal_rP   rŒ   rŸ   rL   r^   rw   rM   rN   ÚdistÚis_availableÚis_initializedre   r�   rŽ   r�   Ú	get_groupÚget_global_rankr¤   Ú	broadcastÚdata)rh   r#   rŠ   rV   Útp_planÚdevice_meshÚpgr¹   s           r$   r£   ÚLoraLayer.reset_lora_parameters  sì  € Ø EÓ)ØŸ{™{×/Ñ/Ó1Ô1Ø$¨Ò,ô —G‘G×,Ñ,¨T¯[©[¸Ñ-F×-MÑ-MÔQU×QZÒQZÐ[\ÓQ]Ð,Ò^Ø&×,Ñ,Ó.°*Ó<Ü—G‘G—O‘O D§K¡K°Ñ$=×$DÑ$DÈ!ÈdÏfÉfÐUaÑNbÑJb�OÒcä$Ð'QÐ?PÑ>RÐ%SÓTÐTÜ—‘—‘˜tŸ{™{¨<Ñ8×?Ñ?Ô@Ø—>‘> ,×/Ü—G‘G—N‘N 4§;¡;¨|Ñ#<×#AÑ#AÔBØ×4Ñ4×9Ñ9Ó;Ó;ô —‘—‘˜t×4Ñ4°\ÑBÔCÜ—‘—‘ × 5Ñ 5°lÑ CÔDØ—>‘> ,×/ä—G‘G—N‘N 4×#8Ñ#8¸Ñ#F×#KÑ#KÔLô ×Ò×Ò¤4×#6Ò#6×#8Ò#8Ø×,Ñ,Ó.ˆJÜ˜j¨-¸Ó>ˆGÜ! *Ð.?ÀÓFˆKØÒ&ØÐ"MÓMÜ—M’MØ# G 9ð -Mð Môð
 !×*Ñ*Ó,�Ü×*Ò*¨2¨qÓ1�Ø˜iÓ'¨L¿K¹KÓ,Gà×>Ñ>¸|ÔLÜ—N’N 4§;¡;¨|Ñ#<×#CÑ#C×#HÑ#HÈcÓ\Ø 	Ó)¨l¿k¹kÓ.Ià×>Ñ>¸|ÔLÜ—N’N 4§;¡;¨|Ñ#<×#CÑ#C×#HÑ#HÈcÓ\ØÐ 3Ó3¸×H]ÑH]Ó8]Ø×>Ñ>¸|ÔLÜ—N’N 4×#8Ñ#8¸Ñ#F×#KÑ#KÐQTÓ_ð 9^Ð3ð# 'ð	 $9Ðr(   c                óâ  • U R                  5       nUR                  n[        U5      nUR                  nU(       a  [	        U5      nOEU[
        R                  [
        R                  [
        R                  4;   a  UnO[        SU S35      eU R                  U   nU R                  U   nUR                  [
        R                  5      n[
        R                  R                  UR                  5      u  pšU	S S 2S U24   U
S U pËUR!                  5       U R"                  U   R                  l        UR!                  5       U R$                  U   R                  l        U=R                  XpR$                  U   R                  -  U R"                  U   R                  -  -  sl        US:X  a[  UR'                  UUR(                  UR*                  UR,                  UR.                  S9R                  UR0                  5      nXbl        g US:X  aE  UR'                  UUR2                  UR4                  S9R                  UR0                  5      nXbl        g UR                  U5      nXbR                  l        g )Nz.Unsupported data type for the base layer. Got Ú.Ú4bit)Ú
quant_typeÚquant_storageÚcompress_statisticsr"   Ú8bit)Úrequires_gradÚhas_fp16_weights)re   r–   r   Údtyper   r_   Úfloat32Úfloat16Úbfloat16Ú	TypeErrorrR   rP   ÚtoÚlinalgÚqrrÄ   Ú
contiguousrK   rL   Ú	__class__rÌ   rÍ   rÎ   r"   ÚdevicerÐ   rÑ   )rh   r#   rV   r,   Úbnb_param_typerÒ   Úweight_tensorÚscale_factorrP   ÚQÚRÚQrÚRrs                r$   rž   ÚLoraLayer.olora_init;  s  € Ø×(Ñ(Ó*ˆ
Ø ×'Ñ'ˆÜ+¨KÓ8ˆØ×!Ñ!ˆæä4°ZÓ@‰MØ”u—}‘}¤e§m¡m´U·^±^ÐDÓDØ'‰MäÐLÈUÈGÐSTÐUÓVÐVà—|‘| LÑ1ˆØ�F‰F�<Ñ ˆØ%×(Ñ(¬¯©Ó7ˆÜ�|‰|�‰˜}×1Ñ1Ó2‰ˆà’1�b�q�b�5‘˜1˜R˜a˜5ˆBà02·±³ˆ�‰�LÑ!×(Ñ(Ô-Ø02·±³ˆ�‰�LÑ!×(Ñ(Ô-à×Ò˜l¯[©[¸Ñ-F×-MÑ-MÑMÐPT×P[ÑP[Ð\hÑPi×PpÑPpÑpÑpÕØ˜VÓ#Ø'×1Ñ1ØØ&×1Ñ1Ø)×7Ñ7Ø$/×$CÑ$CØ"×)Ñ)ð 2ð ÷ ‰b�×#Ñ#Ó$ð ð !.ÕØ˜vÓ%Ø'×1Ñ1ØØ)×7Ñ7Ø!,×!=Ñ!=ð 2ð ÷ ‰b�×#Ñ#Ó$ð	 ð
 !.Õà)×,Ñ,¨UÓ3ˆMØ%2×ÑÕ"r(   c           	     ó@  • U R                  5       R                  nUR                  nU[        R                  [        R
                  [        R                  4;  a  [        S5      e[        UR                  [        R                  5      U R                  5      nUS:X  ay  [        R                  R                  UR                  SS9u  pVnUS S 2S U R                  U   24   nUS U R                  U    n	X�R                  U   -  n	US U R                  U    n
O�[!        UR#                  S5      5      S:X  ab  [%        UR                  U R                  U   ['        UR#                  S5      S   5      S9u  p‰nX�R                  U   -  n	UR)                  5       n
O[+        S	U S
35      e[        R,                  " [        R.                  " U	5      5      U
-  nU[        R,                  " [        R.                  " U	5      5      -  nXÀR0                  U   R                  l        XÐR2                  U   R                  l        UR                  U R                  U   U-  U-  -
  n[        UR                  U5      U R                  5      nX0R                  5       R                  l        g )Nz�Please initialize PiSSA under float32, float16, or bfloat16. Subsequently, re-quantize the residual model to help minimize quantization errors.r   F)Úfull_matricesÚ_niter_é   éÿÿÿÿ)ÚniterzLinit_lora_weights should be 'pissa' or 'pissa_niter_[number of iters]', got ú	 instead.)re   r–   rÒ   r_   rÓ   rÔ   rÕ   rÖ   r   r×   Úfan_in_fan_outrØ   ÚsvdrÄ   rP   rR   ÚlenÚsplitr	   ÚintÚtrŒ   Údiagr˜   rK   rL   )rh   r#   rŠ   r–   rÒ   ÚVÚSÚUhÚVrÚSrÚUhrÚUrrK   rL   s                 r$   r›   ÚLoraLayer.pissa_inith  s2  € Ø×$Ñ$Ó&×-Ñ-ˆØ—‘ˆØœŸ™¬¯©´u·~±~ÐFÓFÜðeóð ô ˜6Ÿ9™9¤U§]¡]Ó3°T×5HÑ5HÓIˆØ Ó'ä—|‘|×'Ñ'¨¯©À5Ð'ÐI‰HˆA�"Ø’1Ð,˜Ÿ™˜|Ñ,Ð,Ð,Ñ-ˆBØÐ)�T—V‘V˜LÑ)Ð*ˆBØ—,‘,˜|Ñ,Ñ,ˆBØÐ+�t—v‘v˜lÑ+Ð,‰CÜÐ"×(Ñ(¨Ó3Ó4¸Ó9Ü$Ø—‘˜TŸV™V LÑ1¼Ð=N×=TÑ=TÐU^Ó=_Ð`bÑ=cÓ9dñ‰JˆB�Bð —,‘,˜|Ñ,Ñ,ˆBØ—$‘$“&‰CäØ^Ð_pÐ^qÐqzÐ{óð ô —’œEŸJšJ r›NÓ+¨cÑ1ˆØ”e—j’j¤§¢¨B£Ó0Ñ0ˆØ06�‰�LÑ!×(Ñ(Ô-Ø06�‰�LÑ!×(Ñ(Ô-Ø—‘˜tŸ|™|¨LÑ9¸FÑBÀVÑKÑKˆÜ˜6Ÿ9™9 UÓ+¨T×-@Ñ-@ÓAˆØ,2×ÑÓ×$Ñ$Õ)r(   c           	     óê  • U R                  5       nUR                  nUR                  nU[        R                  [        R
                  [        R                  4;  a  [        S5      eUR                  [        R                  5      n[        U[        5      (       a7  UR                  R                  S5      nUR                  R                  S5      nO6UR                  R                  S5      nUR                  R                  S5      n[        US5      (       d  [        S5      eUR                  nUR                   n	UR"                  n
UR$                  nU R&                  U   n[        R(                  " U
5      R+                  5       (       d)  [        R,                  " U
5      R+                  5       (       a  [        S5      e[        R(                  " U	5      R+                  5       (       d)  [        R,                  " U	5      R+                  5       (       a  [        S5      e[        R(                  " U5      R+                  5       (       d)  [        R,                  " U5      R+                  5       (       a  [        S5      eU	R                  S5      U:w  d  U	R                  S5      U:w  a#  [        S	U	R                  5        S
U SU S35      eU
R                  S5      U:w  a   [        SU
R                  5        S
U S35      eUR                  S5      U:w  d  UR                  S5      U:w  a#  [        SUR                  5        S
U SU S35      eX R.                  U   -  n
UR1                  5       R3                  U
R5                  5       R7                  SS5      5      R9                  5       nU	R3                  U
R5                  5       5      R9                  5       nXÐR:                  U   R                  l        XàR<                  U   R                  l        U R.                  U   U-  U-  n[?        XðR@                  S9nUR                  U-
  nUR                  U5      nX@R                  5       R                  l        U?g )Nz�Please initialize CorDA under float32, float16, or bfloat16. Subsequently, re-quantize the residual model to help minimize quantization errors.r   r   Úeigensz’`eigens` attribute not found for layer, please run `preprocess_corda` first. More information can be found at examples/corda_finetuning/README.md.zdInvalid value found in matrix S. Please file an issue at https://github.com/huggingface/peft/issues.zdInvalid value found in matrix U. Please file an issue at https://github.com/huggingface/peft/issues.zdInvalid value found in matrix V. Please file an issue at https://github.com/huggingface/peft/issues.zMatrix U size mismatch: z vs. (z, zé). Please make sure the `lora_config` and `model` argument of `preprocess_corda` is consistent with `get_peft_model`. If you're using cache in `preprocess_corda`, please make sure the cache is built with the same model and LoRA rank.zMatrix S size mismatch: zê,). Please make sure the `lora_config` and `model` argument of `preprocess_corda` is consistent with `get_peft_model`. If you're using cache in `preprocess_corda`, please make sure the cache is built with the same model and LoRA rank.zMatrix V size mismatch: ré   )rì   )!re   r–   rÒ   r_   rÓ   rÔ   rÕ   rÖ   r×   r™   r
   rÄ   Úsizer§   rŒ   rü   ÚU_WCÚS_WCÚV_WCrP   ÚisnanÚanyÚisinfrR   rñ   Úmulr˜   ÚviewrÚ   rK   rL   r   rì   )rh   r#   rŠ   Úlinearr–   rÒ   Úout_dimÚin_dimrü   ÚUrô   ró   rP   rK   rL   Údeltas                   r$   rœ   ÚLoraLayer.corda_init‹  s¢  € Ø×$Ñ$Ó&ˆØ—‘ˆØ—‘ˆØœŸ™¬¯©´u·~±~ÐFÓFÜðeóð ð —‘œ5Ÿ=™=Ó)ˆä�fœf×%Ñ%Ø—k‘k×&Ñ& qÓ)ˆGØ—[‘[×%Ñ% aÓ(‰Fà—k‘k×&Ñ& qÓ)ˆGØ—[‘[×%Ñ% aÓ(ˆFô �v˜x×(Ñ(ÜðXóð ð —‘ˆØ�K‰KˆØ�K‰KˆØ�K‰KˆØ�F‰F�<Ñ ˆô �;Š;�q‹>×Ñ×Ñ¤5§;¢;¨q£>×#5Ñ#5×#7Ñ#7ÜØvóð ô �;Š;�q‹>×Ñ×Ñ¤5§;¢;¨q£>×#5Ñ#5×#7Ñ#7ÜØvóð ô �;Š;�q‹>×Ñ×Ñ¤5§;¢;¨q£>×#5Ñ#5×#7Ñ#7ÜØvóð ð
 �6‰6�!‹9˜Ó 1§6¡6¨!£9°£>ÜØ*¨1¯6©6«8¨*°F¸7¸)À2ÀaÀSð Ipð póð ð
 �6‰6�!‹9˜‹>ÜØ*¨1¯6©6«8¨*°F¸1¸#ð >Yð Yóð ð
 �6‰6�!‹9˜Ó !§&¡&¨£)¨q£.ÜØ*¨1¯6©6«8¨*°F¸6¸(À"ÀQÀCð Hpð póð ð 	
�\‰\˜,Ñ'Ñ'ˆð —‘“—‘˜1Ÿ6™6›8Ÿ=™=¨¨QÓ/Ó0×;Ñ;Ó=ˆØ—‘�q—v‘v“x“×+Ñ+Ó-ˆØ06�‰�LÑ!×(Ñ(Ô-Ø06�‰�LÑ!×(Ñ(Ô-ð —‘˜\Ñ*¨VÑ3°fÑ<ˆÜ˜%×0CÑ0CÑDˆØ—‘˜uÑ$ˆØ—‘˜5Ó!ˆØ,2×ÑÓ×$Ñ$Ô)ð ‰Mr(   c                óH  • U R                  5       R                  nUR                  S   U R                  U   UR                  S   S.n[	        U40 UD6u  pVnXR
                  R                  5       ;   a:  X`R
                  U   R                  l        XpR                  U   R                  l        XR                  R                  5       ;   a:  X`R                  U   R                  l        XpR                  U   R                  l        XPR                  5       R                  l        g )NÚ
loftq_bitsÚ
loftq_iter)Únum_bitsÚreduced_rankÚnum_iter)re   r–   Úloftq_configrP   r   rK   r»   rÄ   rL   rM   rN   )rh   r#   ro   r–   r:   ÚqweightrK   rL   s           r$   r   ÚLoraLayer.loftq_initß  sð   € Ø×$Ñ$Ó&×-Ñ-ˆà×+Ñ+¨LÑ9Ø ŸF™F <Ñ0Ø×+Ñ+¨LÑ9ñ
ˆô #-¨VÑ">°vÑ">Ñˆ˜ØŸ;™;×+Ñ+Ó-Ó-à4:�K‰K˜Ñ%×,Ñ,Ô1Ø4:�K‰K˜Ñ%×,Ñ,Ô1Ø×0Ñ0×5Ñ5Ó7Ó7à>D×!Ñ! ,Ñ/×6Ñ6Ô;Ø>D×!Ñ! ,Ñ/×6Ñ6Ô;Ø,3×ÑÓ×$Ñ$Õ)r(   c                óL  • U R                   U   nUS-  S:w  a  [        SU S35      e[        R                  " X"5      n[        R                  R                  U5      u  pEUSS S2S S 24   nUSS S2S S 24   nU R                  5       R                  R                  n[        R                  " U R                  US-  5      R                  U5      R                  S-  n	[        R                  " US-  U R                  5      R                  R                  U5      S-  n
[        R                  " U	R                  5       R!                  U5      5      U R"                  U   l        [        R                  " U
R                  5       R!                  U5      5      U R$                  U   l        g )Nrè   r   zAOrthogonal initialization requires the LoRA rank to be even, got rë   r   g      $@)rP   rŒ   r_   ÚrandnrØ   rÙ   re   r–   rÒ   rf   ÚmmÚTrg   rW   r•   rÚ   r×   rK   rL   )rh   r#   ÚrankÚXrà   Ú_Úq_oddÚq_evenrÒ   rK   rL   s              r$   r    ÚLoraLayer.orthogonal_initò  sS  € ð �v‰v�lÑ#ˆØ�!‰8�q‹=ÜÐ`ÐaeÐ`fÐfoÐpÓqÐqä�KŠK˜Ó#ˆÜ�|‰|�‰˜qÓ!‰ˆØ�!�$�Q�$š�'‘
ˆØ�1�4�a�4š�7‘ˆØ×#Ñ#Ó%×,Ñ,×2Ñ2ˆÜ—’˜T×-Ñ-¨t°q©yÓ9×<Ñ<¸UÓC×EÑEÈÑLˆÜ—’˜T Q™Y¨×(9Ñ(9Ó:×<Ñ<×?Ñ?ÀÓGÈ$ÑNˆÜ+-¯<ª<¸×8IÑ8IÓ8K×8NÑ8NÈuÓ8UÓ+Vˆ�‰�LÑ!Ô(Ü+-¯<ª<¸×8IÑ8IÓ8K×8NÑ8NÈuÓ8UÓ+Vˆ�‰�LÑ!Õ(r(   c           	     ó°  • U R                  5       n[        US5      (       d  U R                  USS9  gUR                  nUc  [	        S5      eUR
                  nUR                  nUR                  nU R                  5       R                  R                  nUR                  [        R                  5      nU R                  5       R                  n	[        X@R                  5      nU R                  U   n
[        R                   " U[#        SU
-  [#        UR$                  5      5      SS9u  p¼nUR'                  5       nUSS2SSU
-  24   nUSSU
-   nUSSU
-  2SS24   nUS	:X  aC  US
SU
-  S2SS24   nUSS2SSU
-  S24   nUSSU
-  S2   nU[        R(                  " U5      -  nOÍUS:X  a7  XêSU
-  2SS24   nUSS2SU
24   nUSU
 nU[        R(                  " U5      -  nO�US:X  a:  USU
2SS24   nUSS2U
SU
-  24   nXÊSU
-   nU[        R(                  " U5      -  nOPUS:X  aJ  [        R*                  " SU
-  5      SU
 nUUSS24   nUU   nUSS2U4   [        R(                  " U5      -  nU R,                  U   nU	R$                  S   nUS:X  a  US-  US-  -  nWU-  nO‡US:X  aq  [        U	R.                  R                  [        R                  5      U R                  5      n[        R                   " UU
SS9u  nnnWS   S:”  a  US   US   -  nWU-  nOUS:X  a
  WU-  nWU-  nWR                  U5      nWR                  U5      nUR1                  5       U R2                  U   R                  l        UR1                  5       U R4                  U   R                  l        [        U	R.                  R                  [        R                  5      U R                  5      nUUR7                  5       UR7                  5       -  -  nUU-
  n[        UR                  U5      U R                  5      nUU R                  5       R                  l        U?g)aÕ  
Initialize LoRA weights using gradient approximation.

Uses SVD on the gradient matrix to initialize adapters in a way that aligns with the direction of full
fine-tuning.

Expects that `preprocess_loraga` has been called before this, which attaches the `loraga_grad` attribute to the
base layer.

If gradients are not available (e.g., when loading from a saved adapter), falls back to gaussian
initialization. The weights will be overwritten by the state_dict anyway.
Ú_peft_loraga_gradT)rŠ   Nz}lora_ga_config must be provided when init_lora_weights='lora_ga'. Please pass lora_ga_config=LoraGAConfig(...) to LoraConfig.é   )Úqrê   rè   ÚArBrr   r   ÚA2rBrÚArB2rÚrandomÚstableg      Ð?g      à?Ú
weight_svdÚgd_scale)re   r§   r£   r   rŒ   Ú	directionÚscaleÚstable_gammar–   rÒ   r×   r_   rÓ   r   rì   rP   r	   ÚminÚshaperñ   rò   ÚrandpermrR   rÄ   rÚ   rK   rL   Úfloat)rh   r#   r¢   rV   Úgradr*  r+  r,  rÒ   r–   rP   r	  rô   ró   ÚVhÚlora_A_weightÚlora_B_weightÚS_BÚindicesÚscaling_factorrg   rß   Úweight_datar  Úweight_SÚweight_offsets                             r$   r¡   ÚLoraLayer.lora_ga_init  sT  € ð ×(Ñ(Ó*ˆ
ô �zÐ#6×7Ñ7ð ×&Ñ& |ÀtÐ&ÑLØà×+Ñ+ˆð Ñ!ÜðNóð ð #×,Ñ,ˆ	Ø×$Ñ$ˆØ%×2Ñ2ˆØ×#Ñ#Ó%×,Ñ,×2Ñ2ˆà�w‰w”u—}‘}Ó%ˆØ×$Ñ$Ó&×-Ñ-ˆä˜×2Ñ2Ó3ˆà�F‰F�<Ñ ˆô ×#Ò# D¬C°°A±´s¸4¿:¹:³Ó,GÈqÑQ‰ˆˆað �S‰S‹UˆàŠa��1�q‘5�ˆj‰MˆØˆg��A‘ˆJˆØ��!�a‘%�š�
‰^ˆà˜Óà˜q 1 q¡5¨1˜}ªaÐ/Ñ0ˆMØša  Q¨¡U¨Q Ð.Ñ/ˆMØ�A˜˜A™ �MÑ"ˆCØ)¬E¯JªJ°s«OÑ;‰Mà˜'Ó!à 1 q¡5˜yª!˜|Ñ,ˆMØša  ! ˜e™HˆMØ�B�Q�%ˆCØ)¬E¯JªJ°s«OÑ;‰Mà˜'Ó!à˜r ˜r¢1˜u™IˆMØša  Q¨¡U ˜l™OˆMØ˜˜A™�,ˆCØ)¬E¯JªJ°s«OÑ;‰Mà˜(Ó"Ü—n’n Q¨¡UÓ+¨B¨QÐ/ˆGØ˜wª˜z™NˆMØ�G‘*ˆCØša ˜j™M¬E¯JªJ°s«OÑ;ˆMàŸ™ lÑ3ˆØ—|‘| A‘ˆà�HÓØ(¨$Ñ.°<ÀÑ3DÑEˆLØ)¨LÑ8‰Mà�lÓ"Ü# F§K¡K§N¡N´5·=±=Ó$AÀ4×CVÑCVÓWˆKÜ"×.Ò.¨{¸aÀqÑI‰NˆAˆx˜Ø�1‰v˜‹zØ'¨™{¨S°©VÑ3�Ø -°Ñ <�øà�jÓ Ø)¨NÑ:ˆMØ)¨NÑ:ˆMð &×(Ñ(¨Ó/ˆØ%×(Ñ(¨Ó/ˆð 1>×0HÑ0HÓ0Jˆ�‰�LÑ!×(Ñ(Ô-Ø0=×0HÑ0HÓ0Jˆ�‰�LÑ!×(Ñ(Ô-ô   §¡§¡¬u¯}©}Ó =¸t×?RÑ?RÓSˆØ&¨-×*=Ñ*=Ó*?À-×BUÑBUÓBWÑ*WÑXˆØ! MÑ1ˆÜ §¡¨uÓ 5°t×7JÑ7JÓKˆØ,7ˆ×ÑÓ×$Ñ$Ô)ð Ñ(r(   c                ó    • X R                   U'   g rl   )ra   ©rh   ÚkeyÚvalues      r$   Ú_cache_storeÚLoraLayer._cache_storey  s   € à!�‰�SÒr(   c                ó<   • U R                   R                  U5      nU$ rl   )ra   Úpopr=  s      r$   Ú
_cache_popÚLoraLayer._cache_pop}  s   € à—‘× Ñ  Ó%ˆØˆr(   c                óH  • XR                   ;  a  gU R                  R                  US5      (       aC  X R                  U   -  [        R
                  " U R                  U   5      -  U R                   U'   gX R                  U   -  U R                  U   -  U R                   U'   g)zªSet the scale of the given adapter to the initial scale multiplied by the provided factor

The initial scale is determined by the configured `r` (rank) and `lora_alpha`.
NF)rR   r]   r‹   rQ   r—   r˜   rP   )rh   r¬   r+  s      r$   Ú	set_scaleÚLoraLayer.set_scale‚  s„   € ð
 Ÿ,™,Ó&àØ�?‰?×Ñ˜w¨×.Ñ.Ø$)¯O©O¸GÑ,DÑ$DÄtÇyÂyÐQU×QWÑQWÐX_ÑQ`ÓGaÑ$aˆD�L‰L˜Ò!à$)¯O©O¸GÑ,DÑ$DÀtÇvÁvÈgÁÑ$VˆD�L‰L˜Ò!r(   c                ó¤   • US:X  a  gU R                    H9  nX R                  R                  5       ;  a  M"  U R                  U==   U-  ss'   M;     g)zHMultiply the current scale of all active adapters by the provided factorr   N)r¦   rK   r»   rR   ©rh   r+  r+   s      r$   Úscale_layerÚLoraLayer.scale_layer�  sF   € à�A‹:Øà"×2Ô2ˆNØ§[¡[×%5Ñ%5Ó%7Ó7Ùà�L‰L˜Ó(¨EÑ1Õ(ò	 3r(   Nc                óÎ  • U R                    HÕ  nX R                  R                  5       ;  a  M"  Uc‘  U R                  R	                  US5      (       aB  U R
                  U   [        R                  " U R                  U   5      -  U R                  U'   Mˆ  U R
                  U   U R                  U   -  U R                  U'   M¶  U R                  U   U-  U R                  U'   M×     g)zËDivide the current scale of all active adapters by the provided factor. If `scale=None` is passed, reset to
initial scale

The initial scale is determined by the configured `r` (rank) and `lora_alpha`.

NF)
r¦   rK   r»   r]   r‹   rQ   r—   r˜   rP   rR   rJ  s      r$   Úunscale_layerÚLoraLayer.unscale_layerš  sÀ   € ð #×2Ô2ˆNØ§[¡[×%5Ñ%5Ó%7Ó7Ùà‰}Ø—?‘?×&Ñ& ~°u×=Ñ=Ø37·?±?À>Ñ3RÔUY×U^ÒU^Ð_c×_eÑ_eÐftÑ_uÓUvÑ3v�D—L‘L Ó0à37·?±?À>Ñ3RÐUY×U[ÑU[Ð\jÑUkÑ3k�D—L‘L Ó0à/3¯|©|¸NÑ/KÈeÑ/S�—‘˜^Ó,ò 3r(   c                ó’  • UR                  SS5      nUc  g[        U5      [        U5      :w  a&  S[        U5       S[        U5       S3n[        U5      eU R                  (       a  Sn[        U5      eU Vs1 s H  ofS:w  d  M
  UiM     nnU H1  nU R                  R                  US5      (       d  M&  S	n[        U5      e   gs  snf )
úMCheck if the arguments are compatible with the configs and state of the modelÚadapter_namesNzNLength of `adapter_names` should be the same as the number of inputs, but got z and z respectively.z`Cannot pass `adapter_names` when there are merged adapters, please call `unmerge_adapter` first.Ú__base__Fz1Cannot pass `adapter_names` when DoRA is enabled.)r‹   rî   rŒ   Úmergedr\   )	rh   r8   Úargsr:   rR  ÚmsgÚnameÚunique_adaptersr#   s	            r$   Ú_check_forward_argsÚLoraLayer._check_forward_args­  sÇ   € àŸ
™
 ?°DÓ9ˆØÑ Øäˆq‹6”S˜Ó'Ó'à`Ü�}Ó%Ð& e¬C°«F¨8°>ðCð ô ˜S“/Ð!à�;�;ð uˆCÜ˜S“/Ð!ñ -:ÓPªM DÀZÑ=OŸ4©MˆÐPÛ+ˆLØ�}‰}× Ñ  ¨u×5Ó5ØI�Ü  “oÐ%ò ,ùò Qs   Á7	CÂCc          	     ó  • [          Vs0 s H  oUUR                  US 5      _M     nnU R                  " U/UQ70 UD6nUR                  n[	        U5      n	/ n
U	 H9  nU
R                  [        U5       VVs/ s H  u  pÍXÛ:X  d  M  UPM     snn5        M;     UR                  SS 5      n[        U	5       GH;  u  nnUS:X  a  M  UU R                  R                  5       ;  a  M/  U R                  U   nU R                  U   nU R                  U   nU R                  U   nXU      R                  UR                  R                  5      nUU R                  ;  a7  U" U" U" U5      5      5      U-  nXzU   ==   UR                  U5      -  ss'   MÜ  Ub  X¯    Vs/ s H  nUU   PM
     snUS'   U R                  U   R                   " U 4UUXzU      S.UDUD6nUR                  U5      XzU   '   GM>     U$ s  snf s  snnf s  snf )Nr   rS  ©r+   r8   r9   )ÚVARIANT_KWARG_KEYSrC  rV   rÒ   ÚsetÚappendÚ	enumerater‹   rK   r»   rL   rS   rR   r×   r–   rd   r;   )rh   r8   rR  rU  r:   ÚkÚvariant_kwargsr9   Útorch_result_dtyperX  Úsub_batch_indices_listr¬   ÚindexÚitemr   Úir+   rK   rL   ÚdropoutrR   Ú	sub_batchÚlora_outputÚjs                           r$   Ú_mixed_batch_forwardÚLoraLayer._mixed_batch_forwardÈ  s!  € õ
 ;MÓMÒ:L°Q˜VŸZ™Z¨¨4Ó0Ò0Ñ:LˆÐMØ—’ Ð4 TÒ4¨VÑ4ˆØ#Ÿ\™\Ðä˜mÓ,ˆØ!#ÐÛ&ˆGØ"×)Ñ)ÄIÈmÔD\Ô*pÒD\±[°UÐ`dÑ`o¯5ÑD\Ò*pÖqñ 'à&×*Ñ*¨?¸DÓAˆÜ!*¨?×!;ÑˆAˆ~Ø Ó+ÙØ T§[¡[×%5Ñ%5Ó%7Ó7Ùà—[‘[ Ñ0ˆFØ—[‘[ Ñ0ˆFØ×'Ñ'¨Ñ7ˆGØ—l‘l >Ñ2ˆGð °Ñ3Ñ4×7Ñ7¸¿¹×8KÑ8KÓLˆIØ T×%6Ñ%6Ó6Ù$¡V©G°IÓ,>Ó%?Ó@À7ÑJ�Ø¨aÑ0Ó1°[·^±^ÐDVÓ5WÑWÕ1à Ñ,ØQgÒQjÓ6kÒQjÈA°}ÀQÔ7GÑQjÑ6k�N ?Ñ3Ø"×/Ñ/°Ñ?×GÒGØðà#1ØØ!¸Ñ";Ñ<ñ	ð
 %ðð ñ�ð 5@·N±NÐCUÓ4V�¨aÑ0Ô1ñ9 "<ð< ˆùòO Nùó +qùò* 7ls   ‰G4Á4G9ÂG9ÆG?)ra   rZ   rV   rc   rb   rf   r:   rK   rL   rQ   r^   rS   rM   rN   r`   rd   r[   rg   rP   rR   r\   r]   )F)rV   ú	nn.Modulerb   Úboolr?   r@   ©r"   rn  r?   z#tuple[int, int] | tuple[None, None]©ro   r   r?   zOptional[LoraVariant]©
r#   r>   rP   rð   rQ   rð   ro   r   r?   r@   )ro   r   )r>  r>   r?  r   r?   r@   )r>  r>   r?   r   )r¬   r>   r+  úfloat | intr?   r@   )r+  rs  r?   r@   rl   )r+  zOptional[float | int]r?   r@   ©
r8   rA   rU  r   rR  ú	list[str]r:   r   r?   rA   )rB   rC   rD   rE   rO   Ú__annotations__rT   ri   r   rp   r­   r£   rž   r›   rœ   r   r_   Úno_gradr    r¡   r@  rD  rG  rK  rN  rY  rl  rH   r0   r(   r$   r=   r=   d   sø   ‡ à+gÐ˜Ógà)WÐ�ÓWö)ô<,ôðlYàðlYð ðlYð ð	lYð
 ðlYð 
ôlYò\2`òh+3òZ!3òFRôh4ð& ‡]‚]ƒ_ñWó ðWò t)ôl"ôô
Wô	2öTò&&ð6,Øð,Ø&)ð,Ø:Cð,ØORð,à	÷,r(   r=   c                  ó    ^ • \ rS rSr   S           SU 4S jjjrSS jrSSS jjrSS jrSS jrSS jr	SSS jjr
SU 4S	 jjrS
rU =r$ )r”   i  c                ó¸   >• [         TU ]  5         [        R                  " X40 UD6  UR                  U l        X l        U R
                  " UU4UUS.UD6  X`l        g )N©rQ   ro   )Úsuperri   r=   rì   Ú_active_adapterr­   Úis_target_conv_1d_layer)	rh   rV   r#   ro   rP   rQ   r}  r:   rÛ   s	           €r$   ri   ÚLinear.__init__  sj   ø€ ô 	‰ÑÔÜ×Ò˜4Ñ6¨vÒ6Ø$×3Ñ3ˆÔà+ÔØ×ÒØØð	
ð "Øñ		
ð
 ò	
ð (?Õ$r(   c                óò   • UR                   b  SSKJn  U" 5       $ UR                  b  SSKJn  U" 5       $ UR
                  S LnUR                  (       d  U(       d  g SSKJnJn  U(       a  U" 5       $ U" 5       $ )Nr   )ÚArrowLinearVariant)ÚBdLoraLinearVariant)ÚALoraLinearVariantÚDoraLinearVariant)	Úarrow_configÚvariantsr€  Ú
use_bdlorar�  Úalora_invocation_tokensr\   r‚  rƒ  )rh   ro   r:   r€  r�  Ú	use_alorar‚  rƒ  s           r$   rp   ÚLinear.resolve_lora_variant  sf   € Ø×ÑÑ*Ý4á%Ó'Ð'à×ÑÑ(Ý5á&Ó(Ð(à×2Ñ2¸$Ð>ˆ	Ø��¦yØçCæÙ%Ó'Ð'á$Ó&Ð&r(   c                óª  • [        X5      nU(       d  gU GH¹  nX0R                  R                  5       ;   d  M#  U R                  5       nU(       Ga‡  UR                  R
                  R                  5       nUR                  nX0R                  ;  a%  U R                  U5      nXWR                  U5      -  nOU R                  U   R                  XU5      n[        R                  " U5      R                  5       (       d  [        SU S35      eXTR                  l        U R                   U   (       a§  [#        USS5      c  [%        S5      eUR&                  U R(                  U   R&                  U R*                  U   -  -   n[        R                  " U5      R                  5       (       d  [        SU S35      eUR                  U5      UR&                  l        OÜX0R                  ;  a1  U R                  U5      nUR                  =R
                  U-  sl        O)U R                  U   R-                  XUR                  5        U R                   U   (       a_  [#        USS5      c  [%        S5      eUR&                  =R
                  U R(                  U   R&                  U R*                  U   -  -  sl        U R.                  R1                  U5        GM¼     g)á  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`list[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
        to `None`.
Nú1NaNs detected in the merged weights. The adapter ú seems to be brokenrw   úRImpossible to merge LoRA with `lora_bias=True` because the base layer has no bias.)r   rK   r»   re   r–   rÄ   ÚclonerÒ   rd   Úget_delta_weightr×   r-   r_   ÚisfiniteÚallrŒ   r^   r�   ÚRuntimeErrorrw   rL   rR   r1   r[   r_  )	rh   Ú
safe_mergerR  r+   rV   r,   Ú
orig_dtypeÚdelta_weightÚnew_biass	            r$   ÚmergeÚLinear.merge1  sn  € ô 0°ÓDˆÞàä+ˆNØ§¡×!1Ñ!1Ó!3Õ3Ø!×0Ñ0Ó2�
ßð #-×"3Ñ"3×"8Ñ"8×">Ñ">Ó"@�KØ!,×!2Ñ!2�JØ%×->Ñ->Ó>Ø'+×'<Ñ'<¸^Ó'L˜Ø#§¡°zÓ'BÑB™à&*×&7Ñ&7¸Ñ&G×&RÑ&RÐSWÐitÓ&u˜ä Ÿ>š>¨+Ó6×:Ñ:×<Ñ<Ü(ØOÐP^ÐO_Ð_rÐsóð ð .9×%Ñ%Ô*à—~‘~ n×5Ü" :¨v°tÓ<ÑDÜ".Ø tó#ð ð $.§?¡?°T·[±[ÀÑ5P×5UÑ5UÐX\×XdÑXdÐesÑXtÑ5tÑ#t˜Ü$Ÿ~š~¨hÓ7×;Ñ;×=Ñ=Ü",Ø"SÐTbÐScÐcvÐ wó#ð ð 08¯{©{¸:Ó/F˜
Ÿ™Ô,øð &×->Ñ->Ó>Ø'+×'<Ñ'<¸^Ó'L˜Ø"×)Ñ)×.Ò.°,Ñ>Ö.à×)Ñ)¨.Ñ9×FÑFÀtÐ]g×]nÑ]nÔoà—~‘~ n×5Ü" :¨v°tÓ<ÑDÜ".Ø tó#ð ð #Ÿ™×,Ò,°·±¸NÑ0K×0PÑ0PÐSW×S_ÑS_Ð`nÑSoÑ0oÑoÕ,à×$Ñ$×+Ñ+¨N×;ò_ ,r(   c                ó   • U R                   (       d  [        R                  " S5        g[        U R                  5      S:”  GaL  U R                  R                  5       nXR                  R                  5       ;   aø  U R                  5       R                  nXR                  ;  aB  UR                  nU R                  U5      nU=R                  UR                  U5      -  sl        O%U R                  U   R                  XU5      nXRl        U R                   U   (       aT  U R                  5       R"                  =R                  U R$                  U   R"                  U R&                  U   -  -  sl        [        U R                  5      S:”  a  GMK  gg©úG
This method unmerges all merged adapter layers from the base weights.
ú Already unmerged. Nothing to do.Nr   ©rT  rŽ   r�   rî   r[   rC  rK   r»   re   r–   rd   rÒ   r�  rÄ   r×   r5   r^   rw   rL   rR   ©rh   r+   r–   r•  r–  Úunmergeds         r$   r5   ÚLinear.unmerget  s-  € ð
 �{�{Ü�MŠMÐ<Ô=ØÜ�$×&Ñ&Ó'¨!Ô+Ø!×1Ñ1×5Ñ5Ó7ˆNØ§¡×!1Ñ!1Ó!3Ó3Ø×,Ñ,Ó.×5Ñ5�Ø!×):Ñ):Ó:Ø!'§¡�JØ#'×#8Ñ#8¸Ó#H�LØ—K’K <§?¡?°:Ó#>Ñ>–Kà#×0Ñ0°Ñ@×HÑHÈÐ_eÓf�HØ"*”Kà—>‘> .×1Ø×'Ñ'Ó)×.Ñ.×3Ò3°t·{±{À>Ñ7R×7WÑ7WÐZ^×ZfÑZfÐguÑZvÑ7vÑvÕ3ô �$×&Ñ&Ó'¨!×+Ð+r(   c                óú  • U R                   U   R                  R                  nU R                   U   R                  R                  nUR                  S:H  =(       a-    U[
        R                  :H  =(       d    U[
        R                  :H  nU R                  U   R                  nU R                   U   R                  nU(       a   UR                  5       nUR                  5       n[        Xe-  U R                  5      U R                  U   -  nU(       ai  UR                  US9nUR                  U5      U R                  U   R                  l        UR                  U5      U R                   U   R                  l        U$ ©úš
Compute the delta weight for the given adapter.

Args:
    adapter (str):
        The name of the adapter for which the delta weight should be computed.
Úcpu©rÒ   )rL   r–   rÜ   rÒ   r�   r_   rÔ   rÕ   rK   r0  r   rì   rR   r×   rÄ   ©rh   r¬   rÜ   rÒ   Úcast_to_fp32Úweight_AÚweight_BÚoutput_tensors           r$   r�  ÚLinear.get_delta_weight‹  s,  € ð —‘˜WÑ%×,Ñ,×3Ñ3ˆØ—‘˜GÑ$×+Ñ+×1Ñ1ˆð
 —{‘{ eÑ+×c°¼%¿-¹-Ñ1G×1bÈ5ÔTY×TbÑTbÑKbˆà—;‘;˜wÑ'×.Ñ.ˆØ—;‘;˜wÑ'×.Ñ.ˆæØ—~‘~Ó'ˆHØ—~‘~Ó'ˆHä! (Ñ"5°t×7JÑ7JÓKÈdÏlÉlÐ[bÑNcÑcˆæØ)×,Ñ,°5Ð,Ð9ˆMð 08¯{©{¸5Ó/AˆD�K‰K˜Ñ ×'Ñ'Ô,Ø/7¯{©{¸5Ó/AˆD�K‰K˜Ñ ×'Ñ'Ô,àÐr(   c           	     ó   • U R                   " U/UQ70 UD6  UR                  SS 5      n[         Vs0 s H  oUUR                  US 5      _M     nnU R                  (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nU$ Ub  U R                  " U/UQ7SU0UDUD6nU$ U R                  (       a  U R                  " U/UQ70 UD6nU$ U R                  " U/UQ70 UD6nUR                  nU R                  R                  5       n	U R                   H»  n
X©;  a  M
  U R                  U
   nU R                  U
   nU R                  U
   nU R                  U
   nU R                  XR                   R                  5      nX R"                  ;  a  X|" U" U" U5      5      5      U-  -   nM•  U R"                  U
   R$                  " U 4U
UUS.UDUD6nM½     UR'                  U5      nU$ s  snf ©NrR  r\  )rY  rC  r]  Údisable_adaptersrT  r5   rV   rl  rÒ   rK   r»   r¦   rL   rS   rR   Ú_cast_input_dtyper–   rd   r;   r×   )rh   r8   rU  r:   rR  ra  rb  r9   rc  Úlora_A_keysr+   rK   rL   rh  rR   s                  r$   r;   ÚLinear.forward­  s  € Ø× Ò  Ð4 TÒ4¨VÒ4ØŸ
™
 ?°DÓ9ˆÝ:LÓMÒ:L°Q˜VŸZ™Z¨¨4Ó0Ò0Ñ:LˆÐMà× × Ø�{�{Ø—‘”Ø—_’_ QÐ8¨Ò8°Ñ8ˆFðB ˆðA Ñ&Ø×.Ò.¨qÐq°4ÒqÀ}ÐqÐXfÐqÐjpÑqˆFð> ˆð= �[�[Ø—_’_ QÐ8¨Ò8°Ñ8ˆFð: ˆð7 —_’_ QÐ8¨Ò8°Ñ8ˆFØ!'§¡ÐàŸ+™+×*Ñ*Ó,ˆKØ"&×"6Ô"6�Ø!Ó4ÙàŸ™ ^Ñ4�ØŸ™ ^Ñ4�Ø×+Ñ+¨NÑ;�ØŸ,™, ~Ñ6�Ø×*Ñ*¨1¯m©m×.AÑ.AÓB�Ø!×):Ñ):Ó:Ø# f©V±G¸A³JÓ-?Ó&@À7Ñ&JÑJ’Fà!×.Ñ.¨~Ñ>×FÒFØðà'5ØØ%ñ	ð
 )ðð !ñ’Fñ #7ð* —Y‘YÐ1Ó2ˆFàˆùòM Ns   ±G;c                ó   • g)NTr0   )rh   r#   s     r$   Úsupports_lora_conversionÚLinear.supports_lora_conversionØ  s   € Ør(   c                ó*   >• [         TU ]  5       nSU-   $ ©Núlora.©r{  Ú__repr__©rh   ÚreprÛ   s     €r$   rº  ÚLinear.__repr__Û  ó   ø€ Ü‰gÑÓ ˆØ˜‰}Ðr(   )r|  rì   r}  ©r   r   F)r#   r>   ro   r   rP   rð   rQ   rð   r}  ro  r?   r@   rq  ©FN©r”  ro  rR  úOptional[list[str]]r?   r@   ©r?   r@   ©r?   rA   ©r8   rA   rU  r   r:   r   r?   rA   )Údefault)r#   r>   r?   ro  ©r?   r>   )rB   rC   rD   rE   ri   rp   r˜  r5   r�  r;   r´  rº  rH   Ú__classcell__©rÛ   s   @r$   r”   r”     s�   ø† ð ØØ(-ð?ð ð?ð ð	?ð
 ð?ð ð?ð "&ð?ð 
÷?ð ?ô0'ö,A<ôFwô. ôD)öV÷õ r(   r”   c                  ó,   ^ • \ rS rSrSrU 4S jrSrU =r$ )Ú_LoraEmbeddingAHolderià  zH
A "fake" module to hold the lora_embedding_A weights for the TP hooks.
c                óD   >• [         TU ]  5         UR                  U l        g rl   )r{  ri   r  r–   )rh   Úlora_embedding_A_weightrÛ   s     €r$   ri   Ú_LoraEmbeddingAHolder.__init__å  s   ø€ Ü‰ÑÔØ-×/Ñ/ˆ�r(   )r–   )rB   rC   rD   rE   rF   ri   rH   rÈ  rÉ  s   @r$   rË  rË  à  s   ø† ñ÷0ó 0r(   rË  c                  óö   ^ • \ rS rSr   S             SU 4S jjjrSS jr          SS jrSSS jjrSS jrSS jr	          SS jr
  S         SS	 jjrSS
 jrSU 4S jjrSrU =r$ )Ú	Embeddingiê  c                ó*  >• UR                   (       a0  [        SUR                    SU R                  R                   S35      e[        T
U ]  5         [        R                  X5        UR                  U l        [        USS 5      n[        USS 5      U l	        S U l
        0 U l        0 U l        UbN  [        (       d  [        S5      eUS:w  a  [        SU S	35      eS
SKJn	  ["        R$                  " X˜   5      U l
        X l        U R)                  UUUUS9  g )Nz
lora_bias=z is not supported for rÊ   r´   rµ   zOTensor Parallel with LoRA is only supported for transformers v5.4.0 and above. r¸   z!Unsupported tensor parallel plan z= for embedding layers. Only "embedding_rowwise" is supported.r   )ÚALL_PARALLEL_STYLESrz  )r^   rŒ   rÛ   rB   r{  ri   r=   rì   r�   rÆ   Útp_layerÚ	input_fnsÚ
output_fnsr   r“  Ú)transformers.integrations.tensor_parallelrÒ  ÚcopyÚdeepcopyr|  r­   )rh   rV   r#   ro   rP   rQ   rŠ   r:   rÅ   rÒ  rÛ   s             €r$   ri   ÚEmbedding.__init__ì  s  ø€ ð ××ä˜z¨&×*:Ñ*:Ð);Ð;QÐRV×R`ÑR`×RiÑRiÐQjÐjkÐlÓmÐmä‰ÑÔÜ×Ñ˜4Ô,Ø$×3Ñ3ˆÔä˜* m°TÓ:ˆä" :Ð/@À$ÓGˆÔØˆŒàˆŒØˆŒàÑß,Ò,Ü"Ð#tÓuÐuàÐ-Ó-Ü Ø7¸°yð A!ð !óð õ
 Vä ŸMšMÐ*=Ñ*FÓGˆDŒMà+ÔØ×ÑØØØ!Øð	 	ò 	
r(   c               ó@   • UR                   (       d  g SSKJn  U" 5       $ )Nr   )ÚDoraEmbeddingVariant)r\   r…  rÛ  )rh   ro   r:   rÛ  s       r$   rp   ÚEmbedding.resolve_lora_variant  s   € Ø��Øå2á#Ó%Ð%r(   c                ó”  ^ ^• UR                   nUR                  nUR                  nUR                  n	UR                  n
US::  a  [        SU 35      eT R                  US9nUb  UT R                  U'   UT R                  U'   UT R                  U'   US:”  a  [        R                  " US9nO[        R                  " 5       nUT R                   U'   [        R                  " UT R                  45      n[        R                  " T R                   U45      n[        R"                  " U5      T R$                  U'   [        R"                  " U5      T R&                  U'   U	T R                  U'   U(       a'  U[(        R*                  " U5      -  T R,                  U'   OX2-  T R,                  U'   UT R                  U'   UR.                  T R.                  U'   US:X  a  T R1                  U5        O1US:X  a  T R3                  US5        OU(       a  T R3                  X5        T R5                  U5        UT R                  ;   a"  T R                  U   R6                  " T 4XS	.UD6  T R9                  T R:                  U
S
9  T R<                  bE  [?        T R$                  U   5      mUU 4S jnUU 4S jnUT R@                  U'   UT RB                  U'   g g )Nr   rv   rz   r{   r|   r‚   r…   Tr†   r‡   c                óR   >• TR                   R                  TU TR                  5      $ rl   )rÓ  Ú_prepare_input_fnrÆ   )ÚinputsÚmodrh   s    €€r$   Úinput_fnÚ(Embedding.update_layer.<locals>.input_fne  s"   ø€ Ø—}‘}×6Ñ6°s¸FÀD×DTÑDTÓUÐUr(   c                óR   >• TR                   R                  TU TR                  5      $ rl   )rÓ  Ú_prepare_output_fnrÆ   )Úoutputsrá  rh   s    €€r$   Ú	output_fnÚ)Embedding.update_layer.<locals>.output_fnh  s"   ø€ Ø—}‘}×7Ñ7¸¸WÀd×FVÑFVÓWÐWr(   )"rS   rŠ   r]   r^   rˆ   rŒ   rp   rd   rP   rQ   rW   r‘   r’   r_   r  rf   rg   r•   rM   rN   r—   r˜   rR   r\   r   r£   r¤   r%   r¥   r¦   rÓ  rË  rÔ  rÕ  )rh   r#   rP   rQ   ro   r:   rS   rŠ   r]   r^   rˆ   rd   r©   r©  rª  râ  rç  rá  s   `                @r$   r­   ÚEmbedding.update_layer$  sy  ù€ ð ×*Ñ*ˆØ"×4Ñ4ÐØ×&Ñ&ˆ
Ø×$Ñ$ˆ	Ø×.Ñ.ˆà�‹6ÜÐ^Ð_`Ð^aÐbÓcÐcà×0Ñ0¸Ð0Ð?ˆØÑ#Ø.:ˆD×Ñ˜lÑ+à ˆ�‰ˆ|ÑØ(2ˆ�‰˜Ñ%Ø˜#ÓÜ!#§¢¨lÑ!;Ñä!#§¢£Ðà*<ˆ×Ñ˜,Ñ'ä—;’;  4×#3Ñ#3Ð4Ó5ˆÜ—;’; × 1Ñ 1°1Ð5Ó6ˆÜ.0¯lªl¸8Ó.Dˆ×Ñ˜lÑ+Ü.0¯lªl¸8Ó.Dˆ×Ñ˜lÑ+Ø'0ˆ�‰�|Ñ$æØ)3´d·i²iÀ³lÑ)BˆD�L‰L˜Ò&à)3©ˆD�L‰L˜Ñ&à(2ˆ�‰˜Ñ%à&,§o¡oˆ�‰�lÑ#à Ó'Ø�O‰O˜LÕ)Ø )Ó+à×&Ñ& |°TÕ:ÞØ×&Ñ& |ÔGð 	×2Ñ2°<Ô@à˜4×,Ñ,Ó,Ø×Ñ˜lÑ+×0Ò0°ÐjÀLÑjÐciÒjà×Ñ˜×-Ñ-¸nÐÑMð �=‰=Ñ$Ü'¨×(=Ñ(=¸lÑ(KÓLˆCöVöXð ,4ˆD�N‰N˜<Ñ(Ø,5ˆD�O‰O˜LÒ)ð %r(   c                óv  • [        X5      nU(       d  gU GHŸ  nX0R                  R                  5       ;   d  M#  U R                  5       nUR                  R
                  nU(       a¾  UR                  R                  R                  5       nX0R                  ;  a#  X`R                  U5      R                  U5      -  nOU R                  U   R                  XU5      n[        R                  " U5      R                  5       (       d  [        SU S35      eXdR                  l        OvX0R                  ;  a>  UR                  =R                  U R                  U5      R                  U5      -  sl        O)U R                  U   R!                  XUR                  5        U R"                  R%                  U5        GM¢     g)r‹  NrŒ  r�  )r   rM   r»   re   r–   rÒ   rÄ   r�  rd   r�  r×   r-   r_   r‘  r’  rŒ   r1   r[   r_  )rh   r”  rR  r+   rV   r•  r,   s          r$   r˜  ÚEmbedding.mergen  sm  € ô 0°ÓDˆÞàä+ˆNØ×!6Ñ!6×!;Ñ!;Ó!=Õ=Ø!×0Ñ0Ó2�
Ø'×.Ñ.×4Ñ4�
Þð #-×"3Ñ"3×"8Ñ"8×">Ñ">Ó"@�KØ%×->Ñ->Ó>Ø#×'<Ñ'<¸^Ó'L×'OÑ'OÐPZÓ'[Ñ[™à&*×&7Ñ&7¸Ñ&G×&RÑ&RÐSWÐitÓ&u˜ä Ÿ>š>¨+Ó6×:Ñ:×<Ñ<Ü(ØOÐP^ÐO_Ð_rÐsóð ð .9×%Ñ%Õ*à%×->Ñ->Ó>Ø"×)Ñ)×.Ò.°$×2GÑ2GÈÓ2W×2ZÑ2ZÐ[eÓ2fÑfÖ.à×)Ñ)¨.Ñ9×FÑFÀtÐ]g×]nÑ]nÔoØ×$Ñ$×+Ñ+¨N×;ò1 ,r(   c                óx  • U R                   (       d  [        R                  " S5        g[        U R                  5      S:”  aù  U R                  R                  5       nU R                  5       R                  R                  nXR                  R                  5       ;   a‚  U R                  5       R                  nXR                  ;  a4  U=R                  U R                  U5      R                  U5      -  sl        O%U R                  U   R                  XU5      nXCl        [        U R                  5      S:”  a  Mø  ggr›  )rT  rŽ   r�   rî   r[   rC  re   r–   rÒ   rM   r»   rd   rÄ   r�  r×   r5   )rh   r+   r•  r–   r   s        r$   r5   ÚEmbedding.unmergeš  sí   € ð �{�{Ü�MŠMÐ<Ô=ØÜ�$×&Ñ&Ó'¨!Ó+Ø!×1Ñ1×5Ñ5Ó7ˆNØ×,Ñ,Ó.×5Ñ5×;Ñ;ˆJØ×!6Ñ!6×!;Ñ!;Ó!=Ó=Ø×,Ñ,Ó.×5Ñ5�Ø!×):Ñ):Ó:Ø—K’K 4×#8Ñ#8¸Ó#H×#KÑ#KÈJÓ#WÑW–Kà#×0Ñ0°Ñ@×HÑHÈÐ_eÓf�HØ"*”Kô �$×&Ñ&Ó'¨!×+r(   c                óZ  • U R                   U   R                  nU R                  U   R                  nUR                  S:H  =(       a-    U[
        R                  :H  =(       d    U[
        R                  :H  nU R                  U   nU R                   U   nU(       a   UR                  5       nUR                  5       n[        Xe-  S5      U R                  U   -  nU(       aK  UR                  US9nUR                  U5      U R                  U'   UR                  U5      U R                   U'   U$ )r¤  r¥  Tr¦  )rN   rÜ   rM   rÒ   r�   r_   rÔ   rÕ   r0  r   rR   r×   r§  s           r$   r�  ÚEmbedding.get_delta_weight¬  s  € ð ×&Ñ& wÑ/×6Ñ6ˆØ×%Ñ% gÑ.×4Ñ4ˆð
 —{‘{ eÑ+×c°¼%¿-¹-Ñ1G×1bÈ5ÔTY×TbÑTbÑKbˆà×(Ñ(¨Ñ1ˆØ×(Ñ(¨Ñ1ˆæØ—~‘~Ó'ˆHØ—~‘~Ó'ˆHä! (Ñ"5°tÓ<¸t¿|¹|ÈGÑ?TÑTˆæØ)×,Ñ,°5Ð,Ð9ˆMð .6¯[©[¸Ó-?ˆD×!Ñ! 'Ñ*Ø-5¯[©[¸Ó-?ˆD×!Ñ! 'Ñ*àÐr(   c          	     ó�  • U R                   " U/UQ70 UD6nU R                  5       n[        U5      n/ nU H9  n	UR                  [	        U5       V
Vs/ s H  u  p«X¹:X  d  M  U
PM     snn
5        M;     [	        U5       H¾  u  pÍUS:X  a  M  XÐR
                  R                  5       ;  a  M,  U R
                  U   R                  nU R                  U   R                  nU R                  U   nXU      nU R                  UU5      nUU-  U-  nUb  UUR                  UR                  5      -  nXXU   ==   U-  ss'   MÀ     U$ s  snn
f )NrS  )rV   Ú_get_embed_scaler^  r_  r`  rM   r»   r  rN   rR   Ú_embedr×   rÒ   )rh   r8   rR  rU  r:   r9   Úembed_scalerX  rd  r¬   re  rf  rg  r+   Úembedding_AÚembedding_BrR   ri  Úafter_AÚadapter_outputs                       r$   rl  ÚEmbedding._mixed_batch_forwardÎ  sR  € ð
 —’ Ð4 TÒ4¨VÑ4ˆð ×+Ñ+Ó-ˆä˜mÓ,ˆØ!#ÐÛ&ˆGØ"×)Ñ)ÄIÈmÔD\Ô*pÒD\±[°UÐ`dÑ`o¯5ÑD\Ò*pÖqñ 'ô "+¨?Ö!;ÑˆAØ Ó+ÙØ×%:Ñ%:×%?Ñ%?Ó%AÓAÙà×/Ñ/°Ñ?×AÑAˆKØ×/Ñ/°Ñ?×AÑAˆKØ—l‘l >Ñ2ˆGð °Ñ3Ñ4ˆIØ—k‘k )¨[Ó9ˆGØ%¨Ñ3°wÑ>ˆNð Ñ&Ø!/°+·.±.À×AUÑAUÓ2VÑ!V�à¨!Ñ,Ó-°Ñ?Õ-ñ) "<ð, ˆùó1 +qs   ÁEÁ!Ec           
     óì   • U R                  5       nUb	  U" U45      n[        R                  " UUUR                  UR                  UR
                  UR                  UR                  S9nUb  U" U5      nU$ )N)Úpadding_idxÚmax_normÚ	norm_typeÚscale_grad_by_freqÚsparse)re   ÚFÚ	embeddingrú  rû  rü  rý  rþ  )rh   Úinputr–   râ  rç  rV   Úoutputs          r$   rò  ÚEmbedding._embedö  s|   € ð ×(Ñ(Ó*ˆ
ØÑÙ˜e˜XÓ&ˆEÜ—’ØØØ"×.Ñ.Ø×(Ñ(Ø ×*Ñ*Ø)×<Ñ<Ø×$Ñ$ñ
ˆð Ñ Ù˜vÓ&ˆFØˆr(   c                óz  • U R                   " U/UQ70 UD6  UR                  SS 5      n[         Vs0 s H  oUUR                  US 5      _M     nnU R                  (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nU$ Ub  U R                  " U/UQ7SU0UD6nU$ U R                  (       a  U R                  " U/UQ70 UD6nU$ U R                  " U/UQ70 UD6nUR                  nU R                  5       n	U R                   GH  n
X R                  ;  a  M  X R                  ;  a¹  U R                  U
   R                  nU R                  U
   R                  nU R                  U
   nU R                   R#                  U
S 5      nU R$                  R#                  U
S 5      nU R'                  XXïS9nUU-  U-  nU	b  UU	R)                  UR                  5      -  nUU-   nMÝ  U R                  U
   R*                  " U 4U
UUS.UDUD6nGM     UR)                  U5      nU$ s  snf )NrR  )râ  rç  r\  )rY  rC  r]  r¯  rT  r5   rV   rl  rÒ   rñ  r¦   rM   rd   r  rN   rR   rÔ  r‹   rÕ  rò  r×   r;   )rh   r8   rU  r:   rR  ra  rb  r9   rc  ró  r+   rô  rõ  rR   râ  rç  rö  r÷  s                     r$   r;   ÚEmbedding.forward  sQ  € à× Ò  Ð4 TÒ4¨VÒ4ØŸ
™
 ?°DÓ9ˆÝ:LÓMÒ:L°Q˜VŸZ™Z¨¨4Ó0Ò0Ñ:LˆÐMØ× × Ø�{�{Ø—‘”Ø—_’_ QÐ8¨Ò8°Ñ8ˆFðV ˆðU Ñ&Ø×.Ò.¨qÐ_°4Ò_À}Ð_ÐX^Ñ_ˆFðR ˆðQ �[�[Ø—_’_ QÐ8¨Ò8°Ñ8ˆFðN ˆðK —_’_ QÐ8¨Ò8°Ñ8ˆFØ!'§¡Ðð ×/Ñ/Ó1ˆKà"&×"6Õ"6�Ø!×)>Ñ)>Ó>Ùà!×):Ñ):Ó:Ø"&×"7Ñ"7¸Ñ"G×"IÑ"I�KØ"&×"7Ñ"7¸Ñ"G×"IÑ"I�KØ"Ÿl™l¨>Ñ:�Gà#Ÿ~™~×1Ñ1°.À$ÓG�HØ $§¡× 3Ñ 3°NÀDÓ I�IØ"Ÿk™k¨!À8˜kÐa�GØ&-°Ñ&;¸wÑ%F�Nð #Ñ.Ø)7¸+¿.¹.È×I]ÑI]Ó:^Ñ)^˜à# nÑ4’Fà!×.Ñ.¨~Ñ>×FÒFØðà'5ØØ%ñ	ð
 )ðð !ñ“Fñ) #7ð8 —Y‘YÐ1Ó2ˆFàˆùò_ Ns   ±H8c                ó*   >• [         TU ]  5       nSU-   $ r·  r¹  r»  s     €r$   rº  ÚEmbedding.__repr__B  r¾  r(   )r|  rÆ   rì   rÔ  rÕ  rÓ  )r   r   T)rV   rn  r#   r>   ro   r   rP   rð   rQ   rð   rŠ   zUnion[bool, str]r?   r@   rq  rr  rÀ  rÁ  rÃ  rÄ  rt  )NN)
r  rA   r–   rA   râ  z4Callable[[tuple[torch.Tensor]], torch.Tensor] | Nonerç  z-Callable[[torch.Tensor], torch.Tensor] | Noner?   rA   rÅ  rÇ  )rB   rC   rD   rE   ri   rp   r­   r˜  r5   r�  rl  rò  r;   rº  rH   rÈ  rÉ  s   @r$   rÐ  rÐ  ê  s.  ø† ð ØØ.2ð.
àð.
ð ð.
ð ð	.
ð
 ð.
ð ð.
ð ,ð.
ð 
÷.
ð .
ô`&ðH6àðH6ð ðH6ð ð	H6ð
 ðH6ð 
ôH6öT*<ôX+ô$ ðD&Øð&Ø&)ð&Ø:Cð&ØORð&à	ô&ðX JNØCGðàðð ðð Gð	ð
 Aðð 
õô.3÷jõ r(   rÐ  c                  óª   ^ • \ rS rSr  S           SU 4S jjjr          SS jrS rSSS jjrSS jrSS jr	SS jr
SU 4S	 jjrS
rU =r$ )Ú_ConvNdiG  c           	     óÔ  >• [         TU ]  5         [        R                  X5        UR                  SS5      (       a  [	        S5      eUR
                  S:”  a  [        R                  " S5        XAR
                  -  S:w  a3  [	        SUR                  R                   SUR
                   S	U S
35      eX l
        UR                  R                  5       U l        U R                  UUUUS9  g )Nrˆ  Fz,aLoRA does not support adapting conv layers.r   zMLoRA adapter added to ConvNd layer with groups > 1. Merging is not supported.r   zTargeting a z with groups=z
 and rank zš. Currently, support is limited to conv layers where the rank is divisible by groups. Either choose a different rank or do not target this specific layer.rz  )r{  ri   r=   r‹   rŒ   ÚgroupsrŽ   r�   rÛ   rB   r|  r–   ÚdimÚ_kernel_dimr­   ©rh   rV   r#   ro   rP   rQ   r:   rÛ   s          €r$   ri   Ú_ConvNd.__init__I  sé   ø€ ô 	‰ÑÔÜ×Ñ˜4Ô,Ø�:‰:�k 5×)Ñ)ÜÐKÓLÐLØ×Ñ˜qÓ Ü�MŠMÐiÔjà× Ñ Ñ  AÓ%ÜØ˜z×3Ñ3×<Ñ<Ð=¸]È:×K\ÑK\ÐJ]Ð]gÐhiÐgjð kWð Wóð ð  ,ÔØ%×,Ñ,×0Ñ0Ó2ˆÔà×ÑØØØ!Øð	 	ò 	
r(   c           	     óÎ  • UR                   nUR                  nUR                  nUR                  n	UR                  n
US::  a  [        SU 35      eU	(       a\  [        U R                  5       SS 5      c@  [        R                  " S[        U R                  5       5      R                   S3[        5        U R                  US9nUb  X°R                  U'   X R                  U'   X0R                   U'   US:”  a  ["        R$                  " US9nO["        R&                  " 5       nXÀR                   U'   U R                  5       nUR(                  nUR*                  nUR,                  n[        U5      nS	U R.                  S
-
  -  =nnU" U R0                  X.UUSS9U R2                  U'   U" X R4                  UUUR6                  U	S9U R8                  U'   X�R                  U'   U(       a'  U[:        R<                  " U5      -  U R>                  U'   OX2-  U R>                  U'   X€R                  U'   UR@                  U R@                  U'   US:X  a  U RC                  U5        O1US:X  a  U RE                  US5        OU(       a  U RE                  X5        U RG                  U5        XR                  ;   a"  U R                  U   RH                  " U 4XS.UD6  U RK                  U RL                  U
S9  g )Nr   rv   rw   rx   ry   rz   r{   r|   ©r   rè   Fr~   )r  rw   r‚   r…   Tr†   r‡   )'rS   rŠ   r]   r^   rˆ   rŒ   r�   re   rŽ   r�   r�   rB   r   rp   rd   rP   rQ   rW   r‘   r’   Úkernel_sizeÚstrideÚpaddingr  rf   rK   rg   r  rL   r—   r˜   rR   r\   r   r£   r¤   r%   r¥   r¦   )rh   r#   rP   rQ   ro   r:   rS   rŠ   r]   r^   rˆ   rd   r©   rV   r  r  r  Ú
conv_layerÚ
out_kernelÚ
out_strides                       r$   r­   Ú_ConvNd.update_layerj  s¯  € ð ×*Ñ*ˆØ"×4Ñ4ÐØ×&Ñ&ˆ
Ø×$Ñ$ˆ	Ø×.Ñ.ˆà�‹6ÜÐ^Ð_`Ð^aÐbÓcÐcæœ' $×"5Ñ"5Ó"7¸ÀÓFÑNÜ�MŠMØMÌdÐSW×SfÑSfÓShÓNi×NrÑNrÐMsð tWð Wäôð ×0Ñ0¸Ð0Ð?ˆØÑ#Ø.:×Ñ˜lÑ+à �‰ˆ|ÑØ(2�‰˜Ñ%Ø˜#ÓÜ!#§¢¨lÑ!;Ñä!#§¢£Ðà*<×Ñ˜,Ñ'à×(Ñ(Ó*ˆ
Ø ×,Ñ,ˆØ×"Ñ"ˆØ×$Ñ$ˆÜ˜*Ó%ˆ
Ø"&¨$×*:Ñ*:¸QÑ*>Ñ"?Ð?ˆ
�ZÙ$.¨t×/?Ñ/?ÀÐQWÐY`ÐglÑ$mˆ�‰�LÑ!Ù$.Ø× Ñ  *¨jÀ×ARÑARÐYbñ%
ˆ�‰�LÑ!ð (1�‰�|Ñ$æØ)3´d·i²iÀ³lÑ)BˆD�L‰L˜Ò&à)3©ˆD�L‰L˜Ñ&à(2�‰˜Ñ%à&,§o¡oˆ�‰�lÑ#à Ó'Ø�O‰O˜LÕ)Ø )Ó+à×&Ñ& |°TÕ:ÞØ×&Ñ& |ÔGð 	×2Ñ2°<Ô@à×,Ñ,Ó,Ø×Ñ˜lÑ+×0Ò0°ÐjÀLÑjÐciÒjà×Ñ˜×-Ñ-¸nÐÒMr(   c                ó,   • SSU R                   S-
  -  -   $ )N)ré   r  r   )r  ©rh   s    r$   Ú_get_dora_factor_viewÚ_ConvNd._get_dora_factor_view´  s   € Ø�t˜t×/Ñ/°!Ñ3Ñ4Ñ4Ð4r(   c                ó  • [        X5      nU(       d  gU GHí  nX0R                  R                  5       ;   d  M#  U R                  5       nUR                  R
                  nUR                  S:”  a  [        S5      eU(       Ga{  UR                  R                  R                  5       nX0R                  ;  a%  U R                  U5      nXgR                  U5      -  nOU R                  U   R                  XU5      n[        R                  " U5      R!                  5       (       d  [#        SU S35      eXdR                  l        U R$                  U   (       a§  ['        USS5      c  [)        S5      eUR*                  U R,                  U   R*                  U R.                  U   -  -   n[        R                  " U5      R!                  5       (       d  [#        SU S35      eUR                  U5      UR*                  l        OëX0R                  ;  a@  U R                  U5      nUR                  =R                  UR                  U5      -  sl        O)U R                  U   R1                  XUR                  5        U R$                  U   (       a_  ['        USS5      c  [)        S5      eUR*                  =R                  U R,                  U   R*                  U R.                  U   -  -  sl        U R2                  R5                  U5        GMð     g)a  
Merge the active adapter weights inside the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`list[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
        to `None`.
Nr   z<Merging is not supported for _ConvNd layers with groups > 1!rŒ  r�  rw   rŽ  )r   rK   r»   re   r–   rÒ   r  r!   rÄ   r�  rd   r�  r×   r-   r_   r‘  r’  rŒ   r^   r�   r“  rw   rL   rR   r1   r[   r_  )	rh   r”  rR  r+   rV   r•  r,   r–  r—  s	            r$   r˜  Ú_ConvNd.merge·  s—  € ô 0°ÓDˆÞàä+ˆNØ§¡×!1Ñ!1Ó!3Õ3Ø!×0Ñ0Ó2�
Ø'×.Ñ.×4Ñ4�
à×$Ñ$ qÓ(ä-Ð.lÓmÐmçð #-×"3Ñ"3×"8Ñ"8×">Ñ">Ó"@�KØ%×->Ñ->Ó>Ø'+×'<Ñ'<¸^Ó'L˜Ø#§¡°zÓ'BÑB™à&*×&7Ñ&7¸Ñ&G×&RÑ&RÐSWÐitÓ&u˜ä Ÿ>š>¨+Ó6×:Ñ:×<Ñ<Ü(ØOÐP^ÐO_Ð_rÐsóð ð .9×%Ñ%Ô*à—~‘~ n×5Ü" :¨v°tÓ<ÑDÜ".Ø tó#ð ð $.§?¡?°T·[±[ÀÑ5P×5UÑ5UÐX\×XdÑXdÐesÑXtÑ5tÑ#t˜Ü$Ÿ~š~¨hÓ7×;Ñ;×=Ñ=Ü",Ø"SÐTbÐScÐcvÐ wó#ð ð 08¯{©{¸:Ó/F˜
Ÿ™Ô,øð &×->Ñ->Ó>Ø'+×'<Ñ'<¸^Ó'L˜Ø"×)Ñ)×.Ò.°,·/±/À*Ó2MÑMÖ.à×)Ñ)¨.Ñ9×FÑFÀtÐ]g×]nÑ]nÔoà—~‘~ n×5Ü" :¨v°tÓ<ÑDÜ".Ø tó#ð ð #Ÿ™×,Ò,°·±¸NÑ0K×0PÑ0PÐSW×S_ÑS_Ð`nÑSoÑ0oÑoÕ,à×$Ñ$×+Ñ+¨N×;òi ,r(   c                ó   • U R                   (       d  [        R                  " S5        g[        U R                  5      S:”  GaL  U R                  R                  5       nXR                  R                  5       ;   aø  U R                  5       R                  nXR                  ;  aB  UR                  nU R                  U5      nU=R                  UR                  U5      -  sl        O%U R                  U   R                  XU5      nXRl        U R                   U   (       aT  U R                  5       R"                  =R                  U R$                  U   R"                  U R&                  U   -  -  sl        [        U R                  5      S:”  a  GMK  ggr›  rž  rŸ  s         r$   r5   Ú_ConvNd.unmergeÿ  s-  € ð �{�{Ü�MŠMÐ<Ô=ØÜ�$×&Ñ&Ó'¨!Ô+Ø!×1Ñ1×5Ñ5Ó7ˆNØ§¡×!1Ñ!1Ó!3Ó3Ø×,Ñ,Ó.×5Ñ5�Ø!×):Ñ):Ó:Ø!'§¡�JØ#'×#8Ñ#8¸Ó#H�LØ—K’K <§?¡?°:Ó#>Ñ>–Kà#×0Ñ0°Ñ@×HÑHÈÐ_eÓf�HØ"*”Kà—>‘> .×1Ø×'Ñ'Ó)×.Ñ.×3Ò3°t·{±{À>Ñ7R×7WÑ7WÐZ^×ZfÑZfÐguÑZvÑ7vÑvÕ3ô �$×&Ñ&Ó'¨!×+Ð+r(   c                óÐ  • U R                   U   R                  R                  nU R                  U   R                  R                  nUR
                  S:H  =(       a-    U[        R                  :H  =(       d    U[        R                  :H  nU R                  U   R                  nU R                   U   R                  nU(       a   UR                  5       nUR                  5       nU R                  5       R                  R                  5       SS S:X  ap  UR                  S5      R                  S5      UR                  S5      R                  S5      -  R                  S5      R                  S5      U R                  U   -  nOtU R                  UR!                  SS5      U5      nU R                  5       R"                  S:”  a  XpR                  U   -  nO"UR!                  SS5      U R                  U   -  nU(       ai  UR%                  US9nUR%                  U5      U R                  U   R                  l        UR%                  U5      U R                   U   R                  l        U$ )	r¤  r¥  rè   r!  )r   r   é   r   r   r¦  )rL   r–   rÜ   rK   rÒ   r�   r_   rÔ   rÕ   r0  re   rý   ÚsqueezeÚ	unsqueezerR   Úconv_fnr   r  r×   rÄ   r§  s           r$   r�  Ú_ConvNd.get_delta_weight  sú  € ð —‘˜WÑ%×,Ñ,×3Ñ3ˆØ—‘˜GÑ$×+Ñ+×1Ñ1ˆð
 —{‘{ eÑ+×c°¼%¿-¹-Ñ1G×1bÈ5ÔTY×TbÑTbÑKbˆà—;‘;˜wÑ'×.Ñ.ˆØ—;‘;˜wÑ'×.Ñ.ˆæØ—~‘~Ó'ˆHØ—~‘~Ó'ˆHð ×ÑÓ ×'Ñ'×,Ñ,Ó.¨q°Ð3°vÓ=à%×-Ñ-¨aÓ0×8Ñ8¸Ó;¸h×>NÑ>NÈqÓ>Q×>YÑ>YÐZ[Ó>\Ñ\×gÑgÐhiÓj×tÑtØóà—‘˜WÑ%ñ&‰Mð !ŸL™L¨×);Ñ);¸A¸qÓ)AÀ8ÓLˆMà×"Ñ"Ó$×+Ñ+¨aÓ/Ø -·±¸WÑ0EÑ E‘à -× 7Ñ 7¸¸1Ó =ÀÇÁÈWÑ@UÑ U�æØ)×,Ñ,°5Ð,Ð9ˆMð 08¯{©{¸5Ó/AˆD�K‰K˜Ñ ×'Ñ'Ô,Ø/7¯{©{¸5Ó/AˆD�K‰K˜Ñ ×'Ñ'Ô,àÐr(   c           	     óø  • U R                   " U/UQ70 UD6  UR                  SS 5      n[         Vs0 s H  oUUR                  US 5      _M     nnU R                  (       a9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nU$ Ub  U R                  " U/UQ7SU0UD6nU$ U R                  (       a  U R                  " U/UQ70 UD6nU$ U R                  " U/UQ70 UD6nUR                  nU R                   HÓ  n	X�R                  R                  5       ;  a  M"  U R                  U	   n
U R                  U	   nU R                  U	   nU R                  U	   nU R                  XR                   R                  5      nX�R"                  ;  a  X{" U
" U" U5      5      5      U-  -   nM­  U R"                  U	   R$                  " U 4U	UUS.UDUD6nMÕ     UR'                  U5      nU$ s  snf r®  )rY  rC  r]  r¯  rT  r5   rV   rl  rÒ   r¦   rK   r»   rL   rS   rR   r°  r–   rd   r;   r×   )rh   r8   rU  r:   rR  ra  rb  r9   rc  r+   rK   rL   rh  rR   s                 r$   r;   Ú_ConvNd.forwardC  s  € Ø× Ò  Ð4 TÒ4¨VÒ4ØŸ
™
 ?°DÓ9ˆÝ:LÓMÒ:L°Q˜VŸZ™Z¨¨4Ó0Ò0Ñ:LˆÐMØ× × Ø�{�{Ø—‘”Ø—_’_ QÐ8¨Ò8°Ñ8ˆFð@ ˆð? Ñ&Ø×.Ò.¨qÐ_°4Ò_À}Ð_ÐX^Ñ_ˆFð< ˆð; �[�[Ø—_’_ QÐ8¨Ò8°Ñ8ˆFð8 ˆð3 —_’_ QÐ8¨Ò8°Ñ8ˆFØ!'§¡Ðà"&×"6Ô"6�Ø!¯©×)9Ñ)9Ó);Ó;ÙØŸ™ ^Ñ4�ØŸ™ ^Ñ4�Ø×+Ñ+¨NÑ;�ØŸ,™, ~Ñ6�Ø×*Ñ*¨1¯m©m×.AÑ.AÓB�à!×):Ñ):Ó:Ø# f©V±G¸A³JÓ-?Ó&@À7Ñ&JÑJ’Fà!×.Ñ.¨~Ñ>×FÒFØðà'5ØØ%ñ	ð
 )ðð !ñ’Fñ #7ð* —Y‘YÐ1Ó2ˆFØˆùòI Ns   ±G7c                ó*   >• [         TU ]  5       nSU-   $ r·  r¹  r»  s     €r$   rº  Ú_ConvNd.__repr__l  r¾  r(   )r|  r  ©r   r   )rV   rn  r#   r>   ro   r   rP   rð   rQ   rð   r?   r@   rr  rÀ  rÁ  rÃ  rÄ  )r8   rA   r?   rA   rÇ  )rB   rC   rD   rE   ri   r­   r  r˜  r5   r�  r;   rº  rH   rÈ  rÉ  s   @r$   r	  r	  G  sº   ø† ð Øð
àð
ð ð
ð ð	
ð
 ð
ð ð
ð 
÷
ð 
ðBHNàðHNð ðHNð ð	HNð
 ðHNð 
ôHNòT5öF<ôPwô,,ô\'÷Rõ r(   r	  c                  ó2   ^ • \ rS rSrU 4S jrSS jrSrU =r$ )ÚConv2diq  c                ó    >• [         TU ]  " U0 UD6  U R                  S:X  d  [        SU R                   35      e[        R
                  U l        g )Nr!  z0Conv2d layer kernel must have 4 dimensions, not )r{  ri   r  rŒ   rÿ  Úconv2dr%  ©rh   rU  r:   rÛ   s      €r$   ri   ÚConv2d.__init__s  óI   ø€ Ü‰Ò˜$Ð) &Ò)Ø×Ñ 1Ó$ÜÐOÐPT×P`ÑP`ÐOaÐbÓcÐcÜ—x‘xˆ�r(   c               ó@   • UR                   (       d  g SSKJn  U" 5       $ )Nr   )ÚDoraConv2dVariant)r\   r…  r4  )rh   ro   r:   r4  s       r$   rp   ÚConv2d.resolve_lora_varianty  ó   € Ø��Øå/á Ó"Ð"r(   ©r%  rq  ©rB   rC   rD   rE   ri   rp   rH   rÈ  rÉ  s   @r$   r-  r-  q  ó   ø† õ ÷#ò #r(   r-  c                  ó2   ^ • \ rS rSrU 4S jrSS jrSrU =r$ )ÚConv1di‚  c                ó    >• [         TU ]  " U0 UD6  U R                  S:X  d  [        SU R                   35      e[        R
                  U l        g )Nr"  z0Conv1d layer kernel must have 3 dimensions, not )r{  ri   r  rŒ   rÿ  Úconv1dr%  r0  s      €r$   ri   ÚConv1d.__init__„  r2  r(   c               ó@   • UR                   (       d  g SSKJn  U" 5       $ )Nr   )ÚDoraConv1dVariant)r\   r…  r@  )rh   ro   r:   r@  s       r$   rp   ÚConv1d.resolve_lora_variantŠ  r6  r(   r7  rq  r8  rÉ  s   @r$   r;  r;  ‚  r9  r(   r;  c                  ó2   ^ • \ rS rSrU 4S jrSS jrSrU =r$ )ÚConv3di“  c                ó    >• [         TU ]  " U0 UD6  U R                  S:X  d  [        SU R                   35      e[        R
                  U l        g )Nr°   z0Conv3d layer kernel must have 5 dimensions, not )r{  ri   r  rŒ   rÿ  Úconv3dr%  r0  s      €r$   ri   ÚConv3d.__init__•  r2  r(   c               ó@   • UR                   (       d  g SSKJn  U" 5       $ )Nr   )ÚDoraConv3dVariant)r\   r…  rH  )rh   ro   r:   rH  s       r$   rp   ÚConv3d.resolve_lora_variant›  r6  r(   r7  rq  r8  rÉ  s   @r$   rC  rC  “  r9  r(   rC  c                  ó  ^ • \ rS rSrSr  S         S U 4S jjjr\S!S j5       r\S"S j5       r\S"S j5       r	\S#S j5       r
\S!S j5       r\S$S	 j5       r\S#S
 j5       r\S!S j5       r\S%S j5       r\S%S j5       r\S&S j5       r\S'S j5       r\S'S j5       rS(S jr\S#S j5       rS)U 4S jjrS*S+S jjrS)S jr        S,S jrS-S jrU 4S jrS.S jr\S 5       rU 4S jrU 4S jr S/U 4S jjr!Sr"U =r#$ )0ÚMultiheadAttentioni¤  a^  LoRA implemented in a multihead attention layer

This is currently only implemented for the case of `_qkv_same_embed_dim = True`, i.e. query, key, and value having
the same dimension.

Note: LoRA is applied to both the in_proj (query/key/value) and out_proj. There is currently no way to specify only
one of them. Don't try to apply LoRA to the out_proj of MultiheadAttention by targeting that layer specifically,
since the forward method of that layer is not being used, hence the LoRA adapter would be ignored.

This is a little bit hacky because of the way that MultiheadAttention is implemented in PyTorch: There are no
`nn.Linear` layers which we can hook onto or, in case of output projection, `.forward` is not used. This
implementation works around these problems by merging the weights before the forward call and unmerging them after
the forward call.
c                ó¬  >• [        USS5      (       d#  [        SU R                  R                   S35      eUR                  (       a"  [        U R                  R                   S35      eUR                  SS5      (       a"  [        U R                  R                   S35      e[        TU ]  5         [        R                  " X40 UD6  [        UR                  [        R                  5      (       a+  [        UR                  U4UUUS	.UD6U R                  l
        O#[        S
U R                  R                   S35      eX l        U R                  X$XSS9  g )NÚ_qkv_same_embed_dimTz?Only same embed for query/key/value is supported as of now for rÊ   z: does not support DoRA (yet), please set use_dora to Falserˆ  Fz< does not support aLoRA (yet), please set use_alora to False)rP   rQ   ro   z.out_proj must be an instance of nn.Linear for rz  )r�   rŒ   rÛ   rB   r\   r‹   r{  ri   r=   r™   Úout_projrW   r”   rV   r|  r­   r  s          €r$   ri   ÚMultiheadAttention.__init__´  s?  ø€ ô �zÐ#8¸$×?Ñ?ô ØQÐRV×R`ÑR`×RiÑRiÐQjÐjkÐlóð ð �?�?ä §¡× 7Ñ 7Ð8Ð8rÐsÓtÐtØ�:‰:�k 5×)Ñ)Ü §¡× 7Ñ 7Ð8Ð8tÐuÓvÐvÜ‰ÑÔÜ×Ò˜4Ñ6¨vÒ6ô �j×)Ñ)¬2¯9©9×5Ñ5Ü'-Ø×#Ñ#Øð(ð Ø%Øñ(ð ñ(ˆD�O‰OÕ$ô ÐMÈdÏnÉn×NeÑNeÐMfÐfgÐhÓiÐià+ÔØ×Ñ˜,°jÐÒPr(   c                ó6   • U R                  5       R                  $ rl   )re   Ú	embed_dimr  s    r$   rQ  ÚMultiheadAttention.embed_dimÜ  ó   € à×"Ñ"Ó$×.Ñ.Ð.r(   c                ó6   • U R                  5       R                  $ rl   )re   Úkdimr  s    r$   rU  ÚMultiheadAttention.kdimà  ó   € à×"Ñ"Ó$×)Ñ)Ð)r(   c                ó6   • U R                  5       R                  $ rl   )re   Úvdimr  s    r$   rY  ÚMultiheadAttention.vdimä  rW  r(   c                ó6   • U R                  5       R                  $ rl   )re   rM  r  s    r$   rM  Ú&MultiheadAttention._qkv_same_embed_dimè  s   € à×"Ñ"Ó$×8Ñ8Ð8r(   c                ó6   • U R                  5       R                  $ rl   )re   Ú	num_headsr  s    r$   r^  ÚMultiheadAttention.num_headsì  rS  r(   c                ó6   • U R                  5       R                  $ rl   )re   rh  r  s    r$   rh  ÚMultiheadAttention.dropoutð  s   € à×"Ñ"Ó$×,Ñ,Ð,r(   c                ó6   • U R                  5       R                  $ rl   )re   Úbatch_firstr  s    r$   rc  ÚMultiheadAttention.batch_firstô  s   € à×"Ñ"Ó$×0Ñ0Ð0r(   c                ó6   • U R                  5       R                  $ rl   )re   Úhead_dimr  s    r$   rf  ÚMultiheadAttention.head_dimø  s   € à×"Ñ"Ó$×-Ñ-Ð-r(   c                ó6   • U R                  5       R                  $ rl   )re   Úin_proj_weightr  s    r$   ri  Ú!MultiheadAttention.in_proj_weightü  s   € à×"Ñ"Ó$×3Ñ3Ð3r(   c                ó6   • U R                  5       R                  $ rl   )re   Úin_proj_biasr  s    r$   rl  ÚMultiheadAttention.in_proj_bias   s   € à×"Ñ"Ó$×1Ñ1Ð1r(   c                óR   • U R                  5       R                  R                  5       $ rl   )re   rN  r  s    r$   rN  ÚMultiheadAttention.out_proj  s    € à×"Ñ"Ó$×-Ñ-×<Ñ<Ó>Ð>r(   c                ó6   • U R                  5       R                  $ rl   )re   Úbias_kr  s    r$   rq  ÚMultiheadAttention.bias_k  ó   € à×"Ñ"Ó$×+Ñ+Ð+r(   c                ó6   • U R                  5       R                  $ rl   )re   Úbias_vr  s    r$   ru  ÚMultiheadAttention.bias_v  rs  r(   c                óB   • U R                  5       R                  " U0 UD6$ rl   )re   Úmerge_masks)rh   rU  r:   s      r$   rx  ÚMultiheadAttention.merge_masks  s!   € Ø×"Ñ"Ó$×0Ò0°$ÐA¸&ÑAÐAr(   c                ó6   • U R                  5       R                  $ rl   )re   Úadd_zero_attnr  s    r$   r{  Ú MultiheadAttention.add_zero_attn  s   € à×"Ñ"Ó$×2Ñ2Ð2r(   c                ór   >• [         TU ]  " U0 UD6  U R                  R                  R                  " U0 UD6  g rl   )r{  r­   rV   rN  r0  s      €r$   r­   ÚMultiheadAttention.update_layer  s2   ø€ Ü‰Ò˜dÐ- fÒ-à�‰× Ñ ×-Ò-¨tÐ>°vÓ>r(   c                ó*  • [        X5      nU(       d  gU GHù  nX0R                  R                  5       ;   d  M#  U R                  5       nUR                  R
                  R                  nU(       Ga‡  UR                  R                  R                  5       R                  5       nX`R                  U5      R                  U5      -  n[        R                  " U5      R                  5       (       d  [!        SU S35      eUR                  R
                  R                  R                  5       R                  5       nXtR                  R                  U5      R                  U5      -  n[        R                  " U5      R                  5       (       d  [!        SU S35      eU?Xdl        UR                  R                  5       ?XtR                  R                  5       l        UR                  R#                  U/S9  OüU R                  U5      R                  U5      nUR                  R                  R                  5       U-   n	U?X”l        UR                  R                  U5      R                  U5      nUR                  R
                  R                  R                  5       U-   n	UR                  R                  5       ?X”R                  R                  5       l        UR                  R#                  U/S9  U R$                  R'                  U5        GMü     g)a  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`List[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
        to `None`.
NrŒ  r�  ©rR  )r   rK   r»   re   rN  r–   rÒ   ri  rÄ   Údetachr�  r�  r×   r_   r‘  r’  rŒ   r˜  r[   r_  )
rh   r”  rR  r+   rV   r•  Úorig_weight_inÚorig_weight_outr–  Úweight_mergeds
             r$   r˜  ÚMultiheadAttention.merge  s”  € ô 0°ÓDˆÞàô ,ˆNØ§¡×!1Ñ!1Ó!3Õ3Ø!×0Ñ0Ó2�
Ø'×0Ñ0×7Ñ7×=Ñ=�
ßð &0×%>Ñ%>×%CÑ%C×%JÑ%JÓ%L×%RÑ%RÓ%T�NØ"×&;Ñ&;¸NÓ&K×&NÑ&NÈzÓ&ZÑZ�NÜ Ÿ>š>¨.Ó9×=Ñ=×?Ñ?Ü(ØOÐP^ÐO_Ð_rÐsóð ð
 '1×&9Ñ&9×&@Ñ&@×&EÑ&E×&LÑ&LÓ&N×&TÑ&TÓ&V�OØ#×':Ñ':×'KÑ'KÈNÓ'[×'^Ñ'^Ð_iÓ'jÑj�OÜ Ÿ>š>¨/Ó:×>Ñ>×@Ñ@Ü(ØOÐP^ÐO_Ð_rÐsóð ð #Ð1Ø0>Ô-à"×+Ñ+×:Ñ:Ó<ÐCØBQ×'Ñ'×6Ñ6Ó8Ô?Ø×'Ñ'×-Ñ-¸^Ð<LÐ-ÒMð $(×#8Ñ#8¸Ó#H×#KÑ#KÈJÓ#W�LØ$.×$=Ñ$=×$BÑ$B×$IÑ$IÓ$KÈlÑ$Z�Mð #Ð1Ø0=Ô-ð $.×#6Ñ#6×#GÑ#GÈÓ#W×#ZÑ#ZÐ[eÓ#f�LØ$.×$7Ñ$7×$>Ñ$>×$CÑ$C×$JÑ$JÓ$LÈ|Ñ$[�MØ"×+Ñ+×:Ñ:Ó<ÐCØBO×'Ñ'×6Ñ6Ó8Ô?Ø×'Ñ'×-Ñ-¸^Ð<LÐ-ÑMØ×$Ñ$×+Ñ+¨N×;ò_ ,r(   c                ó   • U R                   (       d  [        R                  " S5        gU R                  5       nUR                  R
                  R                  R                  n[        U R                  5      S:”  GaZ  U R                  R                  5       nX0R                  R                  5       ;   Ga  U R                  U5      R                  U5      nUR                  R                   U-
  nU?UR#                  S[$        R&                  " USS95        UR                  R                  U5      R                  U5      nUR                  R
                  R                  R                   U-
  nUR                  R
                  ?UR                  R
                  R#                  S[$        R&                  " USS95        [        U R                  5      S:”  a  GMZ  U R                  5       R                  R)                  5         g)rœ  r�  Nr   ri  F©rÐ   r–   )rT  rŽ   r�   re   rN  rV   r–   rÒ   rî   r[   rC  rK   r»   r�  r×   ri  rÄ   Úregister_parameterrW   r•   r5   )rh   rV   r•  r+   r–  Ú
old_weights         r$   r5   ÚMultiheadAttention.unmergec  sŒ  € ð �{�{Ü�MŠMÐ<Ô=Øð ×(Ñ(Ó*ˆ
Ø×(Ñ(×3Ñ3×:Ñ:×@Ñ@ˆ
Ü�$×&Ñ&Ó'¨!Ô+Ø!×1Ñ1×5Ñ5Ó7ˆNØ§¡×!1Ñ!1Ó!3Ô3ð
  $×4Ñ4°^ÓD×GÑGÈ
ÓS�Ø'×6Ñ6×;Ñ;¸lÑJ�
ØÐ-Ø×-Ñ-Ð.>ÄÇÂÈZÐglÑ@mÔnð  *×2Ñ2×CÑCÀNÓS×VÑVÐWaÓb�Ø'×0Ñ0×;Ñ;×BÑB×GÑGÈ,ÑV�
Ø×'Ñ'×2Ñ2Ð9Ø×#Ñ#×.Ñ.×AÑAØœbŸlšl¨:ÀUÑKôô! �$×&Ñ&Ó'¨!Ö+ð( 	×ÑÓ×&Ñ&×.Ñ.Õ0r(   c                ó®  • U(       a  U R                  X#S9  U R                  5       nUR                  nU?UR                  S[        R
                  " UR                  UR                  S95        UR                  R                  5       nUR                  nU?	UR                  S[        R
                  " UR                  UR                  S95        Xdl        U$ )zÔ
Merging and unloading of the MultiheadAttention module

This requires an extra step for MultiheadAttention, which is why there is this special method instead of
relying on the normal merge_and_unload code path.
©r”  rR  ri  r‡  r–   )
r˜  re   ri  rˆ  rW   r•   rÄ   rÐ   rN  r–   )rh   r˜  r”  rR  rV   r–   Úout_proj_layers          r$   Ú"unload_and_optionally_merge_moduleÚ5MultiheadAttention.unload_and_optionally_merge_module„  s¶   € ö Ø�J‰J *ˆJÑJØ×(Ñ(Ó*ˆ
ð ×*Ñ*ˆØÐ%Ø×%Ñ%Ð&6¼¿ºÀVÇ[Á[Ð`f×`tÑ`tÑ8uÔvð $×,Ñ,×;Ñ;Ó=ˆØ×&Ñ&ˆØÐ!Ø×)Ñ)¨(´B·L²LÀÇÁÐ\b×\pÑ\pÑ4qÔrà,ÔØÐr(   c                óž  • U R                   U   R                  R                  nU R                   U   R                  R                  nUR                  S:H  =(       a    U[
        R                  :H  nU R                  U   R                  nU R                   U   R                  nU(       a   UR                  5       nUR                  5       nXe-  U R                  U   -  nU(       ai  UR                  US9nUR                  U5      U R                  U   R                  l        UR                  U5      U R                   U   R                  l        U$ r£  )rL   r–   rÜ   rÒ   r�   r_   rÔ   rK   r0  rR   r×   rÄ   r§  s           r$   r�  Ú#MultiheadAttention.get_delta_weight   s  € ð —‘˜WÑ%×,Ñ,×3Ñ3ˆØ—‘˜GÑ$×+Ñ+×1Ñ1ˆð
 —{‘{ eÑ+×F°¼¿¹Ñ0Fˆà—;‘;˜wÑ'×.Ñ.ˆØ—;‘;˜wÑ'×.Ñ.ˆæØ—~‘~Ó'ˆHØ—~‘~Ó'ˆHà!Ñ,°·±¸WÑ0EÑEˆæØ)×,Ñ,°5Ð,Ð9ˆMð 08¯{©{¸5Ó/AˆD�K‰K˜Ñ ×'Ñ'Ô,Ø/7¯{©{¸5Ó/AˆD�K‰K˜Ñ ×'Ñ'Ô,àÐr(   c                ó€   >• SU;   a#  [        SU R                  R                   S35      e[        TU ]  " U/UQ70 UD6  g )NrR  r¸  z( does not support mixed adapter batches.)rÖ   rÛ   rB   r{  rY  ©rh   r8   rU  r:   rÛ   s       €r$   rY  Ú&MultiheadAttention._check_forward_argsÂ  sB   ø€ Ø˜fÓ$Ü˜e D§N¡N×$;Ñ$;Ð#<Ð<dÐeÓfÐfÜ‰Ò# AÐ7¨Ò7°Ó7r(   c                óf  • UR                   nU R                  " U/UQ70 UD6  U R                  (       a8  U R                  (       a  U R	                  5         U R
                  " U/UQ70 UD6nOõU R                  (       a  U R
                  " U/UQ70 UD6nOÍU R                  5       R                  nUR                  U R                  :w  a6  U R                  5       R                  R                  n[        SU SU S35      eU R                   Vs/ s H  oˆU R                  ;   d  M  UPM     n	n U R                  U	S9  U R
                  " U/UQ70 UD6nU R	                  5         US   R                  U5      US   b  US   R                  U5      4nU$ US   4nU$ s  snf ! U R	                  5         f = f)NzThe out_proj layer of z has merged layers but zJ itself doesn't; please ensure that either both or none have merged layersr€  r   r   )rÒ   rY  r¯  rT  r5   rV   re   rN  r¦   rÛ   rB   rŒ   rK   r˜  r×   )
rh   ÚqueryrU  r:   Úprevious_dtyper9   rN  Úcls_namer±   r¦   s
             r$   r;   ÚMultiheadAttention.forwardÇ  s›  € ØŸ™ˆØ× Ò  Ð8¨Ò8°Ò8à× × Ø�{�{Ø—‘”Ø—_’_ UÐ<¨TÒ<°VÑ<‰FØ�[�[Ø—_’_ UÐ<¨TÒ<°VÑ<‰Fà×*Ñ*Ó,×5Ñ5ˆHØ×'Ñ'¨4×+?Ñ+?Ó?ð  ×.Ñ.Ó0×:Ñ:×CÑC�Ü Ø,¨X¨JÐ6MÈhÈZð XBð Bóð ð +/×*>Ò*>ÓSÒ*> QÀtÇ{Á{ÑBRŸqÑ*>ˆOÐSðØ—
‘
¨�
Ñ9ØŸš¨Ð@°Ò@¸Ñ@�ð —‘”à˜‘)—,‘,˜~Ó.ÐPVÐWXÑPYÑPe°°q±	·±¸^Ó0LÐuˆØˆð lrÐrsÑktÐuˆØˆùò Tøð —‘•ús   ÄFÄ#FÄ,%F ÆF0c                ót  • U R                  5       nUR                  nU?UR                  S[        R                  " UR
                  UR                  S95        UR                  R                  5       nUR                  nU?UR                  S[        R                  " UR
                  UR                  S95        g )Nri  r‡  r–   )	re   ri  rˆ  rW   r•   rÄ   rÐ   rN  r–   )rh   rV   r–   s      r$   Ú_restore_weightsÚ#MultiheadAttention._restore_weightsó  s—   € ð ×(Ñ(Ó*ˆ
Ø×*Ñ*ˆØÐ%Ø×%Ñ%Ð&6¼¿ºÀVÇ[Á[Ð`f×`tÑ`tÑ8uÔvð  ×(Ñ(×7Ñ7Ó9ˆ
Ø×"Ñ"ˆØÐØ×%Ñ% h´·²¸V¿[¹[ÐX^×XlÑXlÑ0mÕnr(   c                óD   >• U R                  5         [        TU ]  " U0 UD6$ rl   )r›  r{  Ú
state_dictr0  s      €r$   rž  ÚMultiheadAttention.state_dict  s$   ø€ Ø×ÑÔÜ‰wÒ! 4Ð2¨6Ñ2Ð2r(   c                óD   >• U R                  5         [        TU ]  " U0 UD6$ rl   )r›  r{  Únamed_modulesr0  s      €r$   r¡  Ú MultiheadAttention.named_modules  s$   ø€ à×ÑÔÜ‰wÒ$ dÐ5¨fÑ5Ð5r(   c                ó*   >• [         TU ]  5       nSU-   $ r·  r¹  r»  s     €r$   rº  ÚMultiheadAttention.__repr__  r¾  r(   )r|  r+  )
r#   r>   ro   r   rP   rð   rQ   rð   r?   r@   )r?   rð   )r?   zOptional[int])r?   ro  )r?   r0  )r?   znn.Parameter)r?   rn  )r?   zOptional[nn.Parameter])r?   z,tuple[Optional[torch.Tensor], Optional[int]]rÃ  rÀ  rÁ  )r˜  ro  r”  ro  rR  rÂ  r?   znn.MultiheadAttentionrÄ  )r–  rA   rU  r   r:   r   r?   rA   rÇ  )$rB   rC   rD   rE   rF   ri   ÚpropertyrQ  rU  rY  rM  r^  rh  rc  rf  ri  rl  rN  rq  ru  rx  r{  r­   r˜  r5   rŽ  r�  rY  r;   r   r›  rž  r¡  rº  rH   rÈ  rÉ  s   @r$   rK  rK  ¤  sä  ø† ñð( Øð&Qð ð&Qð ð	&Qð
 ð&Qð ð&Qð 
÷&Qð &QðP ó/ó ð/ð ó*ó ð*ð ó*ó ð*ð ó9ó ð9ð ó/ó ð/ð ó-ó ð-ð ó1ó ð1ð ó.ó ð.ð ó4ó ð4ð ó2ó ð2ð ó?ó ð?ð ó,ó ð,ð ó,ó ð,ôBð ó3ó ð3÷?ö
E<ôN1ðBØðØ'+ðØ<Oðà	ôô8 õD8ô
'ðX ñoó ðoõ(3õ6÷
õ r(   rK  c                  óB   ^ • \ rS rSrSrU 4S jr\S 5       rS rSr	U =r
$ )Ú_LoraParameterProxyi  z�This proxies an `nn.Parameter` that is targeted with LoRA.
Intended to be used in conjunction with `nn.utils.parametrize`, see `ParamWrapper`.
c                ó.   >• [         TU ]  5         Xl        g rl   )r{  ri   r–  )rh   r–  rÛ   s     €r$   ri   Ú_LoraParameterProxy.__init__  s   ø€ Ü‰ÑÔØ(Õr(   c                ó2  • U R                   nUR                   nU[        ;  a  [        SU S[         S35      eU R                  U5      n X-   n[        R
                  " U5      nUR                  UR                  UR                  S9nUR                  U5      $ )Nz8There is an attempt to upcast the targeted parameter to z but the only supported are: rÊ   )r-  Úmax)	rÒ   r   r“  r×   r_   ÚfinfoÚclampr-  r«  )r8   Úyr•  Úupcast_dtypeÚzÚinfos         r$   Ú_low_prec_addÚ!_LoraParameterProxy._low_prec_add  s•   € ð —W‘Wˆ
Ø—w‘wˆØÔ5Ó5ÜØJÈ<È.ð Y/Ü/EÐ.FÀaðIóð ð �D‰D�ÓˆØ‰Eˆä�{Š{˜:Ó&ˆØ�G‰G˜Ÿ™ d§h¡hˆGÐ/ˆØ�t‰t�JÓÐr(   c                ó–   ^• [        U4S j[         5       5      (       a  U R                  TU R                  5      $ TU R                  -   $ )Nc              3  ó`   >#   • U  H#  n[        [        US 5      TR                  :H  v •  M%     g 7frl   )r�   r_   rÒ   )Ú.0Ú
dtype_nameÚWs     €r$   Ú	<genexpr>Ú._LoraParameterProxy.forward.<locals>.<genexpr>3  s#   øé € Ð[Ê]¸zŒw”u˜j¨$Ó/°1·7±7Ö:Ê]ùs   ƒ+.)r  r   r²  r–  )rh   r¸  s    `r$   r;   Ú_LoraParameterProxy.forward2  s>   ø€ ÜÔ[Í]Ó[×[Ñ[Ø×%Ñ% a¨×):Ñ):Ó;Ð;Ø�4×$Ñ$Ñ$Ð$r(   )r–  )rB   rC   rD   rE   rF   ri   rG   r²  r;   rH   rÈ  rÉ  s   @r$   r§  r§    s+   ø† ñõ)ð ñ ó ð ÷$%ð %r(   r§  c                óˆ   • [        U[        R                  5      (       a  U R                  X5        g U R	                  X5        g rl   )r™   rW   r•   rˆ  Úregister_buffer)r"   rW  r  s      r$   Ú_register_parameter_or_bufferr¾  :  s/   € Ü�!”R—\‘\×"Ñ"Ø×!Ñ! $Õ*à×Ñ˜tÕ'r(   c                  óø   ^ • \ rS rSrSr   S             SU 4S jjjrSS jr          SS jrSSS jjrS r	S r
\SS	 j5       rS
 rSSS jjrSS jrU 4S jrSS jrSS jrSU 4S jjrSrU =r$ )ÚParamWrapperiA  aü  A LoRA wrapper for `nn.Parameter`. This layer is dispatched if users target a parameter directly with
`lora_config.target_parameters`
    Note:
    - When accessing the wrapped nn.Parameter directly, e.g. via `module.weight`, the LoRA weights are *not*
      applied.
    - It is currently not implemented to target multiple parameters on the same module. To achieve this, it is
      currently required to create a separate LoRA adapter (with another adapter name) and activate both at the
      same time.
c                ó°  >• X0l         [        T	U ]	  5         [        R                  " X40 UD6  UR                  (       a#  [        SU R                  R                   S35      eUR                  (       a#  [        SU R                  R                   S35      eUR                  (       a#  [        SU R                  R                   S35      eUR                  (       a#  [        SU R                  R                   S35      eU(       a#  [        SU R                  R                   S35      eUR                  U l        SU l        X l        U R                  UUUUS9  g )	Nr¸  ú& does not work with lora_dropout != 0.z# does not work with fan_in_fan_out.z# does not work with lora_bias=True.z" does not work with use_dora=True.z1 does not work with is_target_conv_1d_layer=True.Frz  )Úparameter_namer{  ri   r=   rS   rŒ   rÛ   rB   rì   r^   r\   Ú_did_swap_in_out_featuresr|  r­   )
rh   rV   r#   rÃ  ro   rP   rQ   r}  r:   rÛ   s
            €r$   ri   ÚParamWrapper.__init__L  s+  ø€ ð -ÔÜ‰ÑÔÜ×Ò˜4Ñ6¨vÒ6à××ô ˜u T§^¡^×%<Ñ%<Ð$=Ð=cÐdÓeÐeØ× × Ü˜u T§^¡^×%<Ñ%<Ð$=Ð=`ÐaÓbÐbØ××Ü˜u T§^¡^×%<Ñ%<Ð$=Ð=`ÐaÓbÐbØ�?�?Ü˜u T§^¡^×%<Ñ%<Ð$=Ð=_Ð`ÓaÐaÞ"Ü˜u T§^¡^×%<Ñ%<Ð$=Ð=nÐoÓpÐpà$×3Ñ3ˆÔØ).ˆÔ&Ø+ÔØ×ÑØØØ!Øð	 	ò 	
r(   c                ó2  • U R                  5       nUR                  S:X  a  UR                  u  p4nOSUR                  S   UR                  S   pTnUR                  S;  a0  [        SU R                  R
                   SUR                   S35      eX0l        XE4$ )Nr"  r   r   )rè   r"  r¸  z was initialized with z9 dimensional Parameter, but only 2d and 3d are supported.)Ú	get_paramÚndimr.  rŒ   rÛ   rB   Únum_experts)rh   r"   ÚparamrÉ  rf   rg   s         r$   r   Ú!ParamWrapper._get_in_out_featuresr  s—   € ð —‘Ó ˆØ�:‰:˜‹?Ø5:·[±[Ñ2ˆK¡là56¸¿¹ÀA¹ÈÏÉÐTUÉ lˆKØ�:‰:˜VÓ#ÜØ˜Ÿ™×/Ñ/Ð0Ð0FÀuÇzÁzÀlð S(ð (óð ð
 'ÔØÐ(Ð(r(   c                óP  • UR                   nUR                  nUR                  nUR                  n	UR                  n
US::  a  [        SU 35      eU R                  US9nUb#  [        SU R                  R                   S35      e[        U R                  5       SS5      nU R                  5       R                  S:H  =(       a    U(       + nU(       a;  U R                  (       d*  U R                  U R                  sU l        U l        S	U l        X R                   U'   X0R"                  U'   US
:”  a#  [        SU R                  R                   S35      e[$        R&                  " 5       nU R                   R)                  [$        R*                  " X05      5        [$        R,                  " U R                  X R.                  -  SS9U R0                  U'   [$        R,                  " X R.                  -  U R                  U	S9U R2                  U'   X�R                  U'   U(       a'  U[4        R6                  " U5      -  U R8                  U'   OX2-  U R8                  U'   X€R                  U'   UR:                  U R:                  U'   [=        U[>        5      (       aU  URA                  S5      (       a?  [C        U R                  5       RD                  5         U RG                  X5        S S S 5        GOý[=        U[>        5      (       aU  URA                  S5      (       a?  [C        U R                  5       RD                  5         U RI                  X5        S S S 5        GO“[=        U[>        5      (       aS  URK                  5       S:X  a?  [C        U R                  5       RD                  5         U RM                  U5        S S S 5        GO+US:X  a>  [C        U R                  5       RD                  5         U RO                  U5        S S S 5        OçUS:X  a7  [$        RP                  RS                  U R2                  U   RD                  5        OªUS:X  a>  [C        U R                  5       RD                  5         U RU                  U5        S S S 5        OfUS:X  aH  [C        U R                  5       RD                  5         U RW                  XRX                  5        S S S 5        OU(       a  U R[                  X5        U R]                  U5        XR^                  ;   a"  U R^                  U   RP                  " U 4SU0UD6  U Ra                  U Rb                  U
S9  g ! , (       d  f       Nk= f! , (       d  f       N|= f! , (       d  f       N�= f! , (       d  f       Nž= f! , (       d  f       N¯= f! , (       d  f       NÀ= f)Nr   rv   rz   r¸  z, does not work with LoRA variants like DoRA.Úis_transposedFr"  Tr{   rÂ  r~   r   r€   r�   r‚   rƒ   r„   r…   ro   r‡   )2rS   rŠ   r]   r^   rˆ   rŒ   rp   rÛ   rB   r�   re   rÇ  rÈ  rÄ  rg   rf   rP   rQ   rW   r’   r“   rX   r”   rÉ  rK   rL   r—   r˜   rR   r\   r™   r>   rš   r   r–   r›   rœ   r�   rž   r   r%   rŸ   r    r¡   r¢   r£   r¤   rd   r¥   r¦   )rh   r#   rP   rQ   ro   r:   rS   rŠ   r]   r^   rˆ   rd   rÍ  Úswap_in_out_featuresr©   s                  r$   r­   ÚParamWrapper.update_layerƒ  sn  € ð ×*Ñ*ˆØ"×4Ñ4ÐØ×&Ñ&ˆ
Ø×$Ñ$ˆ	Ø×.Ñ.ˆà�‹6ÜÐ^Ð_`Ð^aÐbÓcÐcà×0Ñ0¸Ð0Ð?ˆØÑ#Ü˜u T§^¡^×%<Ñ%<Ð$=Ð=iÐjÓkÐkô   × 3Ñ 3Ó 5°ÈÓNˆØ $§¡Ó 0× 5Ñ 5¸Ñ :×QÀMÔ@QÐÞ¨×(F×(FØ26×2CÑ2CÀT×EUÑEUÐ/ˆDÔ˜dÔ/Ø-1ˆDÔ*à �‰ˆ|ÑØ(2�‰˜Ñ%Ø˜#Óô ˜u T§^¡^×%<Ñ%<Ð$=Ð=cÐdÓeÐeä!#§¢£Ðà×Ñ× Ñ ¤§¢°Ð/QÓ!RÔSô %'§I¢I¨d×.>Ñ.>À×DTÑDTÑ@TÐ[`Ñ$aˆ�‰�LÑ!Ü$&§I¢I¨a×2BÑ2BÑ.BÀD×DUÑDUÐ\eÑ$fˆ�‰�LÑ!Ø'0�‰�|Ñ$æØ)3´d·i²iÀ³lÑ)BˆD�L‰L˜Ò&à)3©ˆD�L‰L˜Ñ&à(2�‰˜Ñ%à&,§o¡oˆ�‰�lÑ#ô Ð'¬×-Ñ-Ð2C×2NÑ2NÈw×2WÑ2WÜ" 4×#6Ñ#6Ó#8×#?Ñ#?Õ@Ø—‘ Ô@÷ AÑ@äÐ)¬3×/Ñ/Ð4E×4PÑ4PÐQX×4YÑ4YÜ" 4×#6Ñ#6Ó#8×#?Ñ#?Õ@Ø—‘ Ô@÷ AÑ@äÐ)¬3×/Ñ/Ð4E×4KÑ4KÓ4MÐQXÓ4XÜ" 4×#6Ñ#6Ó#8×#?Ñ#?Õ@Ø—‘ Ô-÷ AÑ@à 'Ó)Ü" 4×#6Ñ#6Ó#8×#?Ñ#?Õ@Ø—‘ Ô-÷ AÐ@à %Ó'Ü�G‰G�N‰N˜4Ÿ;™; |Ñ4×;Ñ;Õ<Ø ,Ó.Ü" 4×#6Ñ#6Ó#8×#?Ñ#?Õ@Ø×$Ñ$ \Ô2÷ AÐ@à )Ó+Ü" 4×#6Ñ#6Ó#8×#?Ñ#?Õ@Ø×!Ñ! ,×0EÑ0EÔF÷ AÐ@æØ×&Ñ& |ÔGà×2Ñ2°<Ô@à×,Ñ,Ó,Ø×Ñ˜lÑ+×0Ò0°ÑO¸fÐOÈÒOà×Ñ˜×-Ñ-¸nÐÒM÷7 AÕ@ú÷ AÕ@ú÷ AÕ@ú÷ AÕ@ú÷
 AÕ@ú÷ AÕ@úsH   ËUÌ6UÎU$Ï#U5Ñ$VÒ(VÕ
UÕ
U!Õ$
U2Õ5
VÖ
VÖ
V%c                ó€  ^• U R                  5       R                  n[        R                  " S5      mU R                  5       nU R                  U R                  -    Hà  n[        XS5      n[        U[        R                  [        R                  [        45      (       d  ME  X;  a  ML  [        U4S jUR                  5        5       5      (       a  Mv  UR                  R                  (       d  UR                  R                  (       a  XQ   R!                  X#R                  S9XQ'   MË  XQ   R!                  U5      XQ'   Mâ     g)zm
Move the adapter of the given name to the device of the base layer. Needs special handling for nn.Parameter
ÚmetaNc              3  ó@   >#   • U  H  oR                   T:H  v •  M     g 7frl   )rÜ   )r¶  r}   rÑ  s     €r$   r¹  ÚEParamWrapper._move_adapter_to_device_of_base_layer.<locals>.<genexpr>ç  s   øé € ÐHÒ-G¨—8‘8˜tÖ#Ò-Gùó   ƒr¦  )rÇ  rÜ   r_   rO   rT   r�   r™   rW   rX   rY   r   r  Ú
parametersrÒ   Úis_floating_pointÚ
is_complexr×   )rh   r#   rÜ   rÊ  Úadapter_layer_nameÚadapter_layerrÑ  s         @r$   r¤   Ú2ParamWrapper._move_adapter_to_device_of_base_layerÙ  sï   ø€ ð —‘Ó!×(Ñ(ˆÜ�|Š|˜FÓ#ˆØ—‘Ó ˆà"&×":Ñ":¸T×=SÑ=SÔ"SÐÜ# D¸dÓCˆMÜ˜m¬b¯m©m¼R×=MÑ=MÌzÐ-Z×[Ñ[ÙØÓ0ÙÜÔH¨]×-EÑ-EÔ-GÓH×HÑHÙà�{‰{×,×,°·±×0F×0FØ.;Ñ.I×.LÑ.LÈV×[fÑ[fÐ.LÐ.g�Ó+à.;Ñ.I×.LÑ.LÈVÓ.T�Ó+ò #Tr(   c                óN   • [        U R                  5       U R                  5      nU$ rl   )r�   re   rÃ  )rh   rÊ  s     r$   rÇ  ÚParamWrapper.get_paramï  s"   € Ü˜×+Ñ+Ó-¨t×/BÑ/BÓCˆØˆr(   c                óö  • U R                   S:X  a  [        R                  " X/UQ70 UD6nOæU R                  U   R                  nU R
                  U   R                  nUR                  U R                   SUR                  S   5      nUR                  UR                  S   SU R                   5      nU R                  (       d(  [        R                  " SXe5      U R                  U   -  nO'[        R                  " SXe5      U R                  U   -  nU R                  5       nUR                  [        ;   a(  UR                  UR                   UR                  5      nU$ UR                  UR                   5      nU$ )Nr   ré   r   zo r e, e r i -> e i ozo r e, e r i -> e o i)rÉ  r”   r�  rK   r–   rL   Úreshaper.  rÄ  r_   ÚeinsumrR   rÇ  rÒ   r   r×   rÜ   )rh   r#   rU  r:   r–  r©  rª  rÊ  s           r$   r�  ÚParamWrapper.get_delta_weightó  s=  € Ø×Ñ˜qÓ ä!×2Ò2°4ÐWÈÒWÐPVÑW‰Là—{‘{ <Ñ0×7Ñ7ˆHØ—{‘{ <Ñ0×7Ñ7ˆHà×'Ñ'¨×(8Ñ(8¸"¸h¿n¹nÈRÑ>PÓQˆHà×'Ñ'¨¯©°qÑ(9¸2¸t×?OÑ?OÓPˆHà×1×1Ü$Ÿ|š|Ð,CÀXÓXÐ[_×[gÑ[gÐhtÑ[uÑu‘ô  %Ÿ|š|Ð,CÀXÓXÐ[_×[gÑ[gÐhtÑ[uÑu�à—‘Ó ˆØ�;‰;Ô0Ó0Ø'Ÿ?™?¨5¯<©<¸¿¹ÓEˆLð
 Ðð (Ÿ?™?¨5¯<©<Ó8ˆLØÐr(   c              #  óô  ^ #   • U(       a  [        U 4S jU 5       5      (       d  S v •  g S nU H?  nUT R                  ;  a  M  Uc  T R                  U5      nM+  UT R                  U5      -   nMA     T R                  5       nT R	                  5       R
                  n[        R                  R                  R                  UT R                  [        U5      5        UR                  T R                     R                  R                  U5         [        R                  R                  R                  5          S v •  S S S 5        T R!                  5         g ! , (       d  f       N= f! T R!                  5         f = f7f)Nc              3  ó@   >#   • U  H  oTR                   ;   v •  M     g 7frl   )rK   )r¶  r¬   rh   s     €r$   r¹  Ú.ParamWrapper._activate_lora.<locals>.<genexpr>	  s   øé € Ð)`ÒP_ÀW°T·[±[Ö*@ÒP_ùrÔ  )r  rK   r�  re   rÇ  rÐ   rW   ÚutilsÚparametrizeÚregister_parametrizationrÃ  r§  ÚparametrizationsÚoriginalÚrequires_grad_ÚcachedÚ_remove_parametrizations)rh   r¦   r–  r+   rV   Úrequires_grad_befores   `     r$   Ú_activate_loraÚParamWrapper._activate_lora	  s'  øé € æ¤cÔ)`ÑP_Ó)`×&`Ñ&`ãØàˆÛ-ˆNØ T§[¡[Ó0ÙØÑ#Ø#×4Ñ4°^ÓD’à+¨d×.CÑ.CÀNÓ.SÑS’ñ .ð ×(Ñ(Ó*ˆ
Ø#Ÿ~™~Ó/×=Ñ=ÐÜ
�‰×Ñ×5Ñ5Ø˜×+Ñ+Ô-@ÀÓ-Nô	
ð 	×#Ñ# D×$7Ñ$7Ñ8×AÑA×PÑPÐQeÔfð	,Ü—‘×%Ñ%×,Ñ,Õ.Û÷ /ð ×)Ñ)Õ+÷ /Õ.ûð ×)Ñ)Õ+üs6   ƒDE8Ä(E# Ä4EÄ9E# ÅE8Å
E ÅE# Å#E5Å5E8c                ó¸  • U R                  5       nU R                  nX!R                  ;  a  [        S5      eUR                  U   n[	        U5      S:X  a)  [
        R                  R                  R                  XSS9  g [        [        [	        U5      5      5      nU H   nX5   n[        U[        5      (       d  M  X5	   g    [        R                  " SU  S35        g )NzbSomething went wrong, please report this issue on PEFT: https://github.com/huggingface/peft/issuesr   F)Úleave_parametrizedz+Could not find any LoRA parametrization on z], please open an issue on https://github.com/huggingface/peft/issues and report this warning.)re   rÃ  rç  rŒ   rî   rW   rä  rå  Úremove_parametrizationsÚreversedÚranger™   r§  rŽ   r�   )rh   rV   rÃ  Ú
param_listÚreversed_indicesrg  r"   s          r$   rë  Ú%ParamWrapper._remove_parametrizations+	  sÕ   € à×(Ñ(Ó*ˆ
Ø×,Ñ,ˆØ×!<Ñ!<Ó<ÜØtóð ð  ×0Ñ0°Ñ@ˆ
Üˆz‹?˜aÓä�H‰H× Ñ ×8Ñ8¸ÐhmÐ8ÑnØô $¤E¬#¨j«/Ó$:Ó;ÐÛ!ˆAØ‘]ˆFÜ˜&Ô"5×6Ó6Ø�MÙñ	 "ô �MŠMØ=¸d¸Vð DVð Võr(   c                ój  • [        X5      nU(       d  g U GH  nX0R                  R                  5       ;   d  M#  U R                  5       n[	        X@R
                  5      nU(       a‰  UR                  R                  5       nUR                  nU R                  U5      nXhR                  U5      -  n[        R                  " U5      R                  5       (       d  [        SU S35      eXel        O&U R                  U5      nU=R                  U-  sl        U R                  R!                  U5        GM     g )NrŒ  r�  )r   rK   r»   re   r�   rÃ  rÄ   r�  rÒ   r�  r×   r_   r‘  r’  rŒ   r[   r_  )	rh   r”  rR  r+   rV   rÊ  r,   r•  r–  s	            r$   r˜  ÚParamWrapper.mergeK	  sÿ   € ä/°ÓDˆÞàä+ˆNØ§¡×!1Ñ!1Ó!3Õ3Ø!×0Ñ0Ó2�
Ü 
×,?Ñ,?Ó@�Þð #(§*¡*×"2Ñ"2Ó"4�KØ!,×!2Ñ!2�JØ#'×#8Ñ#8¸Ó#H�LØ§?¡?°:Ó#>Ñ>�Kä Ÿ>š>¨+Ó6×:Ñ:×<Ñ<Ü(ØOÐP^ÐO_Ð_rÐsóð ð "-•Jð $(×#8Ñ#8¸Ó#H�LØ—J’J ,Ñ.•Jà×$Ñ$×+Ñ+¨N×;ò/ ,r(   c                óö  • U R                   (       d  [        R                  " S5        g [        U R                  5      S:”  a¸  U R                  R                  5       nXR                  R                  5       ;   ae  [        U R                  5       U R                  5      nUR                  nU R                  U5      nU=R                  UR                  U5      -  sl        [        U R                  5      S:”  a  M·  g g )Nr�  r   )rT  rŽ   r�   rî   r[   rC  rK   r»   r�   re   rÃ  rÒ   r�  rÄ   r×   )rh   r+   rÊ  r•  r–  s        r$   r5   ÚParamWrapper.unmergek	  s³   € à�{�{Ü�MŠMÐ<Ô=ØÜ�$×&Ñ&Ó'¨!Ó+Ø!×1Ñ1×5Ñ5Ó7ˆNØ§¡×!1Ñ!1Ó!3Ó3Ü × 3Ñ 3Ó 5°t×7JÑ7JÓK�Ø"Ÿ[™[�
Ø#×4Ñ4°^ÓD�Ø—
’
˜lŸo™o¨jÓ9Ñ9•
ô �$×&Ñ&Ó'¨!×+r(   c                ó¢   >• UR                  SS5      (       a#  [        SU R                  R                   S35      e[        TU ]  " U/UQ70 UD6  g)rQ  rR  Nr¸  z, does not support mixed adapter batches yet.)r‹   rŒ   rÛ   rB   r{  rY  r“  s       €r$   rY  Ú ParamWrapper._check_forward_argsx	  sK   ø€ à�:‰:�o t×,Ñ,Ü˜u T§^¡^×%<Ñ%<Ð$=Ð=iÐjÓkÐkÜ‰Ò# AÐ7¨Ò7°Ó7r(   c                óü   • U R                   nU(       aX  U R                  X#S9  [        U[        5      (       a2  UR                  X#S9  UR                   n[        U[        5      (       a  M2  U$ U R	                  5       nU$ )NrŒ  )rV   r˜  r™   rÀ  re   )rh   r˜  r”  rR  rV   s        r$   rŽ  Ú/ParamWrapper.unload_and_optionally_merge_module~	  sr   € Ø—_‘_ˆ
æØ�J‰J *ˆJÑJÜ˜Z¬×6Ñ6Ø× Ñ ¨JÐ ÑTØ'×2Ñ2�
ô ˜Z¬×6Ó6ð
 Ðð ×,Ñ,Ó.ˆJØÐr(   c                ó   • U R                   " U/UQ70 UD6  UR                  SS 5      nU R                  (       a9  U R                  (       a  U R	                  5         U R
                  " U/UQ70 UD6nU$ Ub#  [        SU R                  R                   S35      eU R                  (       a  U R
                  " U/UQ70 UD6nU$ U R                  U R                  5         U R
                  " U/UQ70 UD6nS S S 5        U$ ! , (       d  f       W$ = f)NrR  r¸  z' does not support mixed batch inference)rY  rC  r¯  rT  r5   rV   rŒ   rÛ   rB   rí  r¦   )rh   r8   rU  r:   rR  r9   s         r$   r;   ÚParamWrapper.forwardŠ	  sö   € Ø× Ò  Ð4 TÒ4¨VÒ4ØŸ
™
 ?°DÓ9ˆà× × Ø�{�{Ø—‘”Ø—_’_ QÐ8¨Ò8°Ñ8ˆFð ˆð Ñ&Ü˜u T§^¡^×%<Ñ%<Ð$=Ð=dÐeÓfÐfØ�[�[Ø—_’_ QÐ8¨Ò8°Ñ8ˆFð ˆð ×$Ñ$ T×%9Ñ%9Õ:ØŸš¨Ð<¨TÒ<°VÑ<�÷ ;àˆ÷ ;Ô:àˆús   ÃC>Ã>
Dc                ó„   >• [         TU ]  5       nUR                  S5      S-   nUS U  SU R                   SXS   3nSU-   $ )NÚ(r   z
  parameter_name='z',r¸  )r{  rº  ÚfindrÃ  )rh   r¼  ÚidxrÛ   s      €r$   rº  ÚParamWrapper.__repr__›	  sV   ø€ Ü‰gÑÓ ˆØ�h‰h�s‹m˜aÑˆð �T�c��Ð/°×0CÑ0CÐ/DÀBÀsÈ4ÀyÀkÐRˆØ˜‰}Ðr(   )r|  rÄ  rì   rf   rÉ  rg   rÃ  r¿  )r#   r>   rÃ  r>   ro   r   rP   rð   rQ   rð   r}  ro  r?   r@   rp  rr  rl   )r#   r>   rÜ   zOptional[torch.device]r?   r@   )r¦   ru  rÀ  rÁ  rÃ  )r˜  ro  r”  ro  rR  rÂ  rÅ  rÇ  )rB   rC   rD   rE   rF   ri   r   r­   r¤   rÇ  r�  r   rí  rë  r˜  r5   rY  rŽ  r;   rº  rH   rÈ  rÉ  s   @r$   rÀ  rÀ  A  sù   ø† ñð  ØØ(-ð$
ð ð$
ð ð	$
ð
 ð$
ð ð$
ð ð$
ð "&ð$
ð 
÷$
ð $
ôL)ð"TNàðTNð ðTNð ð	TNð
 ðTNð 
ôTNölUò,òð6 ó,ó ð,ò8ö@<ô@:õ8ô
ô÷"õ r(   rÀ  c                ó  • S n[        U [        5      (       a  U R                  5       nOU nUb  [        X4X2S.UD6nU$ [        U[        R
                  R                  5      (       a  [        X4SU0UD6nU$ [        U[        R
                  R                  5      (       a  [        X4SU0UD6nU$ [        U[        R
                  R                  5      (       a  [        X4SU0UD6nU$ [        U[
        R                  5      (       a  [        X4SU0UD6nU$ [        U[        R
                  R                  5      (       a  [        X4SU0UD6nU$ [        U[        R
                  R                  5      (       a>  UR                  (       a  [        R                  " S5        SUl        [        X4SU0UD6nU$ [        U[        5      (       a=  UR                  (       d  [        R                  " S5        SUl        [        X4SUS.UD6nU$ )N)rÃ  ro   ro   zjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.Fzafan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.T)r}  ro   )r™   r   re   rÀ  r_   rW   rÐ  r-  rC  r;  rK  r”   rì   rŽ   r�   r
   )Útargetr#   ro   rÃ  r:   Ú
new_moduleÚtarget_base_layers          r$   Údispatch_defaultr
  ¤	  sú  € ð €Jä�&œ.×)Ñ)Ø"×1Ñ1Ó3Ñà"ÐàÑ!Ü! &ÐoÀ~ÑoÐhnÑoˆ
ð6 Ðô5 
Ð%¤u§x¡x×'9Ñ'9×	:Ñ	:Ü˜vÑM¸FÐMÀfÑMˆ
ð2 Ðô1 
Ð%¤u§x¡x§¡×	7Ñ	7Ü˜FÑJ¸ÐJÀ6ÑJˆ
ð. Ðô- 
Ð%¤u§x¡x§¡×	7Ñ	7Ü˜FÑJ¸ÐJÀ6ÑJˆ
ð* Ðô) 
Ð%¤r§y¡y×	1Ñ	1Ü˜FÑJ¸ÐJÀ6ÑJˆ
ð& Ðô% 
Ð%¤u§x¡x×'BÑ'B×	CÑ	CÜ'¨ÑVÀVÐVÈvÑVˆ
ð" Ðô! 
Ð%¤u§x¡x§¡×	7Ñ	7Ø× × Ü�MŠMð3ôð %*ˆFÔ!Ü˜FÑJ¸ÐJÀ6ÑJˆ
ð Ðô 
Ð%¤v×	.Ñ	.Ø×$×$Ü�MŠMØsôð %)ˆFÔ!Ü˜FÐhÈ$ÐW]ÑhÐagÑhˆ
àÐr(   rl   )
r  ztorch.nn.Moduler#   r>   ro   r   rÃ  zOptional[str]r?   zOptional[torch.nn.Module])AÚ
__future__r   r×  r—   rŽ   Úcollections.abcr   Ú
contextlibr   Útypingr   r   r   r_   Útorch.distributedÚdistributedr¾   Útorch.nnrW   Útorch.nn.functionalÚ
functionalrÿ  r	   Útransformers.pytorch_utilsr
   Úpeft.import_utilsr   Úpeft.tuners._buffer_dictr   Úpeft.tuners.tuners_utilsr   r   r   Ú
peft.utilsr   r   Úpeft.utils.integrationsr   r   r   r   Úpeft.utils.loftq_utilsr   Úpeft.utils.otherr   Úpeft.utils.warningr   ro   r   r]  r   r=   ÚModuler”   rË  rÐ  r	  r-  r;  rC  rK  r§  r¾  rÀ  r
  r0   r(   r$   Ú<module>r     sk  ðõ #ã Û Û Ý $Ý %ß 'Ñ 'ã Ý  Ý ß Ð Ý Ý -å 7Ý /ß bÑ bß <÷ó õ .Ý &Ý *å ð &Ð&Ð ÷/"ñ /"ôdP
�ô P
ôz\ˆR�Y‰Y˜	ô \ô~0˜BŸI™Iô 0ôZ�—	‘	˜9ô Zôz
gˆb�i‰i˜ô gôT	#ˆWô #ô"#ˆWô #ô"#ˆWô #ô"o˜Ÿ™ Iô oôd%˜"Ÿ)™)ô %òH(ô`�2—9‘9˜iô `ðN %)ð	*Øð*àð*ð ð*ð "ð	*ð ö*r(   