ó
    >:j¸�  ã                  ó  • S SK Jr  S SKrS SKrS SKJrJr  S SKrS SKJ	r	  S SKJ
r
  S SKJr  S SKJr  SS	KJr  SS
KJrJr  SSKJrJrJrJrJr  SSKJrJrJrJrJrJ r J!r!   " S S\ 5      r" " S S\ 5      r# " S S\#5      r$ " S S\ 5      r% " S S\%5      r& " S S\%5      r' " S S\%5      r( " S S\ 5      r) " S S\ 5      r* S'         S(S jjr+S'S)S  jjr, S*     S+S! jjr-S,S" jr. " S# S$\
R^                  5      r0 " S% S&\ 5      r1g)-é    )ÚannotationsN)ÚAnyÚOptional)Úis_xpu_available)Únn)ÚBdLoraConfig)Ú	transposeé   )ÚArrowLoraLinearLayer)Ú
LoraConfigÚ
PeftConfig)ÚDoraConv1dLayerÚDoraConv2dLayerÚDoraConv3dLayerÚDoraEmbeddingLayerÚDoraLinearLayer)ÚConv1dÚConv2dÚConv3dÚ	EmbeddingÚLinearÚLoraVariantÚ_ConvNdc                  óŽ   • \ rS rSr\S	S j5       r\          S
S j5       r \SS j5       r\SS j5       r\SS j5       r	Sr
g)ÚArrowLinearVarianté!   c                ó  • UR                   nUc  [        S5      e[        U R                  US9R	                  U R
                  R                  5      n[        U S5      (       d  [        R                  " 5       U l
        XPR                  U'   g)aß  
Initialise the ArrowLoraLinearLayer() inside lora_arrow. lora_arrow is nn.ModuleDict(), serving as a container
for ArrowLoraLinearLayer(). A layer of the base model with LoRA adapter loaded on it will be like:
----------------------------------------------------
     (qkv_proj): lora.Linear4bit or lora.Linear(
        (base_layer): Linear4bit or Linear (lora_dropout): ModuleDict( ... ) (lora_A): ModuleDict( ... )
        (lora_B): ModuleDict( ... ) (lora_embedding_A): ParameterDict( ... ) (lora_embedding_B): ParameterDict(
        ... ) (lora_magnitude_vector): ModuleDict( ... ) (lora_arrow): ModuleDict(
            (arrow_router): ArrowLoraLinearLayer() )
    )
----------------------------------------------------

Args:
    module (Linear): LoRA Layer of the model, containing base_layer, lora_A, lora_B, etc.
    adapter_name (str): name of the adapter that will be put in lora_arrow.
    The adapter_name is "arrow_router" by default, set in create_arrow_model() in ./arrow.py
Nz9ArrowLinearVariant.init() did not receive an arrow_config)Úin_featuresÚarrow_configÚ
lora_arrow)r   Ú
ValueErrorr   r   ÚtoÚweightÚdeviceÚhasattrr   Ú
ModuleDictr    )ÚmoduleÚadapter_nameÚconfigÚkwargsr   Úarrow_layers         ÚV/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/lora/variants.pyÚinitÚArrowLinearVariant.init"   s}   € ð( ×*Ñ*ˆØÑÜÐXÓYÐYô +Ø×*Ñ*Ø%ñ
÷ ‰"ˆV�]‰]×!Ñ!Ó
"ð 	ô �v˜|×,Ñ,Ü "§¢£ˆFÔà*5×Ñ˜,Ò'ó    c               ó*  • U R                   U   nUR                  U R                  U R                  5        UR	                  U R                  U R                  5        U" UU R                  U R                  U R
                  U   U R                  S9nX6-   $ )uÂ  
Parameters mirror those in PEFTâ€™s `LoraVariant.forward`. Called every time the host Linear does a fwd pass.

build_prototypes() and gen_know_sub() should run only once before routing. Both are implemented in
ArrowLoraLinearLayer (see ./arrow.py). They are lazily invoked in the forward pass below. Attributes of
ArrowLoraLinearLayer() class ensure they execute only a single time.

Args:
    module (Linear): LoRA Layer of the model
    active_adapter (str): name of the arrow route, which should be active to perform arrow.
    x (torch.Tensor): input to the layer
    result (torch.Tensor): output of the base layer.

Return value:
    output of the base model + delta weight computed by arrow layer.
)Úlora_AÚlora_BÚdropoutÚscaling)r    Úgen_know_subr1   r2   Úbuild_prototypesÚlora_dropoutr4   )r'   Úactive_adapterÚxÚresultr*   ÚarrowÚdeltas          r,   ÚforwardÚArrowLinearVariant.forwardF   s„   € ð2 ×!Ñ! .Ñ1ˆð 	×Ñ˜6Ÿ=™=¨&¯-©-Ô8ð 	×Ñ˜vŸ}™}¨f¯m©mÔ<ñ ØØ—=‘=Ø—=‘=Ø×'Ñ'¨Ñ7Ø—N‘Nñ
ˆð ‰~Ðr/   c                ó   • [        S5      e©Nz=Cannot merge an active Arrow router adapter. Remove it first.©ÚRuntimeError©r'   r8   Úorig_weights      r,   Ú
merge_safeÚArrowLinearVariant.merge_safex   ó   € äÐZÓ[Ð[r/   c                ó   • [        S5      er@   rA   rC   s      r,   Úmerge_unsafeÚArrowLinearVariant.merge_unsafe|   rG   r/   c                ó   • [        S5      e)Nz?Cannot unmerge an active Arrow router adapter. Remove it first.rA   rC   s      r,   ÚunmergeÚArrowLinearVariant.unmerge€   s   € äÐ\Ó]Ð]r/   © N)r'   r   r(   Ústrr)   r   ©
r'   r   r8   rO   r9   útorch.Tensorr:   rQ   ÚreturnrQ   ©r'   r   r8   rO   rD   rQ   rR   rQ   ©r'   r   r8   rO   rD   rQ   rR   ÚNone)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ústaticmethodr-   r=   rE   rI   rL   Ú__static_attributes__rN   r/   r,   r   r   !   s¡   † Øó!6ó ð!6ðF ð)Øð)ð ð)ð ð	)ð
 ð)ð 
ó)ó ð)ðVð ó\ó ð\ð ó\ó ð\ð ó^ó ó^r/   r   c                  óŒ   • \ rS rSr\S	S j5       r\S
S j5       r\SS j5       r\S
S j5       r\          SS j5       r	Sr
g)ÚDoraLinearVarianté…   c                óP  • U R                   (       d  U R                  S S  S-   U l        [        [        U SS5      S9nU R                  U   R
                  nU R                  U   R
                  nU R                  =(       a9    UR                  R                  S:H  =(       d    UR                  R                  S:H  nU R                  (       a�  UR                  R                  S;   a  UR                  UR                  5      nOgUR                  R                  S;  a2  [        5       (       a  UR                  S5      nOUR                  S5      nUR                  UR                  5      nU R                  U   nUR                  U R                  5       UUUUS	9  X0R                   U'   g )
N©Úlora_magnitude_vectorÚfan_in_fan_outF©rb   Úcpu)ÚcudaÚxpurf   re   )Ú
base_layerr1   r2   r4   Úplace_on_cpu)ra   Úadapter_layer_namesr   Úgetattrr1   r#   r2   Úephemeral_gpu_offloadr$   Útyper"   r   r4   Úupdate_layerÚget_base_layer)r'   r(   r*   Ú
dora_layerr1   r2   rh   r4   s           r,   r-   ÚDoraLinearVariant.init†   sX  € à×+×+à)/×)CÑ)CÁAÐ)FÐIcÑ)cˆFÔ&ä$´G¸FÐDTÐV[Ó4\Ñ]ˆ
Ø—‘˜|Ñ,×3Ñ3ˆØ—‘˜|Ñ,×3Ñ3ˆØ×3Ñ3×t¸¿¹×9KÑ9KÈuÑ9T×9sÐX^×XeÑXe×XjÑXjÐnsÑXsˆØ×'×'Ø�}‰}×!Ñ! _Ó4ØŸ™ 6§=¡=Ó1‘à—=‘=×%Ñ%¨_Ó<Ü'×)Ñ)Ø!'§¡¨5Ó!1™à!'§¡¨6Ó!2˜ØŸ™ 6§=¡=Ó1�Ø—.‘. Ñ.ˆØ×ÑØ×,Ñ,Ó.ØØØØ%ð 	 ñ 	
ð 6@×$Ñ$ \Ò2r/   c                óœ  • UR                   nU R                  U5      nU R                  U   R                  U[	        X@R
                  5      SS9R                  5       nU R                  U S3U5        U R                  U   R                  U-  n[	        UR                  SS5      U R
                  5      nXbU-   -  nUR                  U5      nU$ ©Nr
   ©r4   ú-weight_norméÿÿÿÿ)ÚdtypeÚget_delta_weightra   Úget_weight_normr	   rb   ÚdetachÚ_cache_storer#   Úviewr"   ©r'   r8   rD   Ú
orig_dtypeÚdelta_weightÚweight_normÚdora_factorÚ
new_weights           r,   rE   ÚDoraLinearVariant.merge_safe¤   sÊ   € à ×&Ñ&ˆ
Ø×.Ñ.¨~Ó>ˆð ×(Ñ(¨Ñ8ß‰_˜[¬)°L×BWÑBWÓ*XÐbcˆ_Ðdß‰V‹Xð 	ð 	×Ñ˜~Ð.¨lÐ;¸[ÔIØ×2Ñ2°>ÑB×IÑIÈKÑWˆÜ × 0Ñ 0°°QÓ 7¸×9NÑ9NÓOˆØ °,Ñ$>Ñ?ˆ
Ø—]‘] :Ó.ˆ
ØÐr/   c                óº  • UR                   nU R                  U5      nU R                  U   R                  U[	        X@R
                  5      SS9R                  5       nU R                  U S3U5        U R                  U   R                  U-  n[	        UR                  SS5      U R
                  5      nXbR                  U-   -  nUR                  U5      nXrl
        g rr   )rv   rw   ra   rx   r	   rb   ry   rz   r#   r{   Údatar"   r|   s           r,   rI   ÚDoraLinearVariant.merge_unsafe¹   sÎ   € à ×&Ñ&ˆ
Ø×.Ñ.¨~Ó>ˆà×(Ñ(¨Ñ8ß‰_˜[¬)°L×BWÑBWÓ*XÐbcˆ_Ðdß‰V‹Xð 	ð 	×Ñ˜~Ð.¨lÐ;¸[ÔIØ×2Ñ2°>ÑB×IÑIÈKÑWˆÜ × 0Ñ 0°°QÓ 7¸×9NÑ9NÓOˆØ ×$4Ñ$4°|Ñ$CÑDˆ
Ø—]‘] :Ó.ˆ
Ø%Õr/   c                ó  • UR                   nU R                  U5      nU R                  U S35      nU R                  U   R                  U-  nUR
                  UR                  SS5      -  U-
  nUR                  U5      nU$ )Nrt   ru   r
   ©rv   rw   Ú
_cache_popra   r#   r„   r{   r"   r|   s           r,   rL   ÚDoraLinearVariant.unmergeÌ   s†   € à ×&Ñ&ˆ
Ø×.Ñ.¨~Ó>ˆØ×'Ñ'¨>Ð*:¸,Ð(GÓHˆØ×2Ñ2°>ÑB×IÑIÈKÑWˆØ ×%Ñ%¨×(8Ñ(8¸¸QÓ(?Ñ?À,ÑNˆ
Ø—]‘] :Ó.ˆ
ØÐr/   c                óJ  • U R                   U   nU R                  U   nU R                  U   nU R                  U   n[	        U[
        R                  5      (       d  U R                  (       d  Un	O
U" U5      nS n	X0R                  U   " UUUUU R                  5       U	US9-   nU$ ©N)r1   r2   r4   rg   Úbase_resultr(   ©
r1   r2   r7   r4   Ú
isinstancer   ÚIdentityÚtrainingra   rn   ©
r'   r8   r9   r:   r*   r1   r2   r3   r4   rŒ   s
             r,   r=   ÚDoraLinearVariant.forwardÖ   ó§   € ð —‘˜~Ñ.ˆØ—‘˜~Ñ.ˆØ×%Ñ% nÑ5ˆØ—.‘. Ñ0ˆä�gœrŸ{™{×+Ñ+°6·?·?Ø ‰Ká˜“
ˆAØˆKà×6Ñ6°~ÒFØØØØØ×,Ñ,Ó.Ø#Ø'ñ
ñ 
ˆð ˆr/   rN   N)r'   r   r(   rO   r*   r   rR   rU   rS   rT   rP   ©rV   rW   rX   rY   rZ   r-   rE   rI   rL   r=   r[   rN   r/   r,   r]   r]   …   s”   † Øó@ó ð@ð: óó ðð( ó&ó ð&ð$ óó ðð ðØðàðð ðð ð	ð 
óó ór/   r]   c                  óŒ   • \ rS rSr\S	S j5       r\S
S j5       r\SS j5       r\S
S j5       r\          SS j5       r	Sr
g)ÚDoraEmbeddingVariantéõ   c                ó  • U R                   c  U R                  S S  S-   U l        [        SS9nU R                  U   nU R                  U   nU R
                  U   nUR                  U R                  5       XEUS9  X0R                   U'   g )Nr`   Trc   ©rg   r1   r2   r4   )ra   ri   r   Úlora_embedding_AÚlora_embedding_Br4   rm   rn   )r'   r(   r*   ro   rš   r›   r4   s          r,   r-   ÚDoraEmbeddingVariant.initö   s—   € à×'Ñ'Ñ/à)/×)CÑ)CÁAÐ)FÐIcÑ)cˆFÔ&ä'°tÑ<ˆ
Ø!×2Ñ2°<Ñ@ÐØ!×2Ñ2°<Ñ@ÐØ—.‘. Ñ.ˆØ×ÑØ×,Ñ,Ó.Ð7GÐjqð 	 ñ 	
ð 6@×$Ñ$ \Ò2r/   c                ó`  • UR                   nU R                  U5      nU R                  U   R                  X$R                  SS9R                  5       nU R                  U S3U5        U R                  U   R                  U-  nUR                  SS5      nXbU-   -  nUR                  U5      nU$ rr   )
rv   rw   ra   rx   ÚTry   rz   r#   r{   r"   r|   s           r,   rE   ÚDoraEmbeddingVariant.merge_safe  s³   € à ×&Ñ&ˆ
Ø×.Ñ.¨~Ó>ˆð ×(Ñ(¨Ñ8ß‰_˜[¯.©.À!ˆ_ÐDß‰V‹Xð 	ð 	×Ñ˜~Ð.¨lÐ;¸[ÔIØ×2Ñ2°>ÑB×IÑIÈKÑWˆØ!×&Ñ& q¨"Ó-ˆØ °,Ñ$>Ñ?ˆ
Ø—]‘] :Ó.ˆ
ØÐr/   c                ó~  • UR                   nU R                  U5      nU R                  U   R                  X$R                  SS9R                  5       nU R                  U S3U5        U R                  U   R                  U-  nUR                  SS5      nXbR                  U-   -  nUR                  U5      nXrl	        g rr   )rv   rw   ra   rx   rž   ry   rz   r#   r{   r„   r"   r|   s           r,   rI   Ú!DoraEmbeddingVariant.merge_unsafe  s·   € à ×&Ñ&ˆ
Ø×.Ñ.¨~Ó>ˆà×(Ñ(¨Ñ8ß‰_˜[¯.©.À!ˆ_ÐDß‰V‹Xð 	ð 	×Ñ˜~Ð.¨lÐ;¸[ÔIØ×2Ñ2°>ÑB×IÑIÈKÑWˆØ!×&Ñ& q¨"Ó-ˆØ ×$4Ñ$4°|Ñ$CÑDˆ
Ø—]‘] :Ó.ˆ
Ø%Õr/   c                ó  • UR                   nU R                  U5      nU R                  U S35      nU R                  U   R                  U-  nUR
                  UR                  SS5      -  U-
  nUR                  U5      nU$ )Nrt   r
   ru   r‡   r|   s           r,   rL   ÚDoraEmbeddingVariant.unmerge-  s†   € à ×&Ñ&ˆ
Ø×.Ñ.¨~Ó>ˆØ×'Ñ'¨>Ð*:¸,Ð(GÓHˆØ×2Ñ2°>ÑB×IÑIÈKÑWˆØ ×%Ñ%¨×(8Ñ(8¸¸BÓ(?Ñ?À,ÑNˆ
Ø—]‘] :Ó.ˆ
ØÐr/   c           
     ó\  • U R                   U   R                  nU R                  U   R                  nU R                  U   nU R                  U   " UUUUU R                  5       U R                  US9u  p‰U R                  5       n
U
b  XšR                  U	R                  5      -  n	Xƒ-  U	-   nU$ )N)r1   r2   r4   rg   Úembed_fnr(   )
rš   rž   r›   r4   ra   rn   Ú_embedÚ_get_embed_scaler"   rv   )r'   r8   r9   r:   r*   Úembedding_AÚembedding_Br4   Úmag_norm_scaleÚdora_resultÚembed_scales              r,   r=   ÚDoraEmbeddingVariant.forward7  s·   € ð ×-Ñ-¨nÑ=×?Ñ?ˆØ×-Ñ-¨nÑ=×?Ñ?ˆØ—.‘. Ñ0ˆà&,×&BÑ&BÀ>Ò&RØØØØØ×,Ñ,Ó.Ø—]‘]Ø'ñ'
Ñ#ˆð ×-Ñ-Ó/ˆØÑ"Ø%¯©°{×7HÑ7HÓ(IÑIˆKàÑ(¨;Ñ6ˆØˆr/   rN   N)r'   r   r(   rO   r*   r   rR   rU   )r'   r   r8   rO   rD   rQ   rR   rQ   )r'   r   r8   rO   rD   rQ   rR   rU   )
r'   r   r8   rO   r9   rQ   r:   rQ   rR   rQ   r”   rN   r/   r,   r–   r–   õ   s”   † Øó@ó ð@ð óó ðð( ó&ó ð&ð$ óó ðð ðØðàðð ðð ð	ð 
óó ór/   r–   c                  óŒ   • \ rS rSr\S	S j5       r\S
S j5       r\SS j5       r\S
S j5       r\          SS j5       r	Sr
g)Ú_DoraConvNdVariantiX  c                ó(  • U R                   c  U R                  S S  S-   U l        U R                  U   R                  nU R                  U   R                  nU R
                  U   nUR                  U R                  5       X4US9  X R                   U'   g )Nr`   r™   )ra   ri   r1   r#   r2   r4   rm   rn   )r'   r(   ro   r1   r2   r4   s         r,   Úinit_convd_variantÚ%_DoraConvNdVariant.init_convd_variantY  sŠ   € à×'Ñ'Ñ/à)/×)CÑ)CÁAÐ)FÐIcÑ)cˆFÔ&à—‘˜|Ñ,×3Ñ3ˆØ—‘˜|Ñ,×3Ñ3ˆØ—.‘. Ñ.ˆØ×Ñ¨6×+@Ñ+@Ó+BÈ6ÐjqÐÑrØ5?×$Ñ$ \Ò2r/   c                ó^  • UR                   nU R                  U5      nU R                  U   R                  X$SS9R	                  5       nU R                  U S3U5        U R                  U   R                  U-  nUR                  " U R                  5       6 X$-   -  nUR                  U5      nU$ ©Nr
   rs   rt   )
rv   rw   ra   rx   ry   rz   r#   r{   Ú_get_dora_factor_viewr"   r|   s           r,   rE   Ú_DoraConvNdVariant.merge_safee  s¶   € à ×&Ñ&ˆ
Ø×.Ñ.¨~Ó>ˆð ×(Ñ(¨Ñ8×HÑHÈÐlmÐHÐn×uÑuÓwð 	ð 	×Ñ˜~Ð.¨lÐ;¸[ÔIØ×2Ñ2°>ÑB×IÑIÈKÑWˆØ ×%Ò% v×'CÑ'CÓ'EÐFÈ+ÑJdÑeˆ
Ø—]‘] :Ó.ˆ
ØÐr/   c                ó~  • UR                   nU R                  U5      nU R                  U   R                  X$SS9R	                  5       nU R                  U S3U5        U R                  U   R                  U-  nUR                  " U R                  5       6 UR                  U-   -  nUR                  U5      nXrl	        g r´   )rv   rw   ra   rx   ry   rz   r#   r{   rµ   r„   r"   r|   s           r,   rI   Ú_DoraConvNdVariant.merge_unsafew  s¿   € à ×&Ñ&ˆ
Ø×.Ñ.¨~Ó>ˆð ×(Ñ(¨Ñ8×HÑHÈÐlmÐHÐn×uÑuÓwð 	ð 	×Ñ˜~Ð.¨lÐ;¸[ÔIØ×2Ñ2°>ÑB×IÑIÈKÑWˆØ ×%Ò% v×'CÑ'CÓ'EÐFÈ+×JZÑJZÐ]iÑJiÑjˆ
Ø—]‘] :Ó.ˆ
Ø%Õr/   c                ó  • UR                   nU R                  U5      nU R                  U S35      nU R                  U   R                  U-  nUR
                  UR                  " U R                  5       6 -  U-
  nUR                  U5      nU$ )Nrt   )	rv   rw   rˆ   ra   r#   r„   r{   rµ   r"   r|   s           r,   rL   Ú_DoraConvNdVariant.unmergeˆ  sŽ   € à ×&Ñ&ˆ
Ø×.Ñ.¨~Ó>ˆØ×'Ñ'¨>Ð*:¸,Ð(GÓHˆØ×2Ñ2°>ÑB×IÑIÈKÑWˆØ ×%Ñ%¨×(8Ò(8¸&×:VÑ:VÓ:XÐ(YÑYÐ\hÑhˆ
Ø—]‘] :Ó.ˆ
ØÐr/   c                óJ  • U R                   U   nU R                  U   nU R                  U   nU R                  U   n[	        U[
        R                  5      (       d  U R                  (       d  Un	O
U" U5      nS n	X0R                  U   " UUUUU R                  5       U	US9-   nU$ r‹   r�   r‘   s
             r,   r=   Ú_DoraConvNdVariant.forward’  r“   r/   rN   N)r'   r   r(   rO   ro   ú	nn.ModulerR   rU   )r'   r   r8   rO   rD   rQ   rR   rQ   )r'   r   r8   rO   rD   rQ   rR   rU   )
r'   r   r8   rO   r9   rQ   r:   rQ   rR   rQ   )rV   rW   rX   rY   rZ   r±   rE   rI   rL   r=   r[   rN   r/   r,   r¯   r¯   X  s”   † Øó	@ó ð	@ð óó ðð" ó&ó ð&ð  óó ðð ðØðàðð ðð ð	ð 
óó ór/   r¯   c                  ó(   • \ rS rSr\SS j5       rSrg)ÚDoraConv1dVarianti±  c                ó>   • [        SS9n[        R                  XUS9  g ©NFrc   )ro   )r   r¯   r±   ©r'   r(   r)   r*   ro   s        r,   r-   ÚDoraConv1dVariant.init²  ó   € ä$°EÑ:ˆ
Ü×-Ñ-¨fÈzÐ-ÒZr/   rN   N)
r'   r   r(   rO   r)   r   r*   r   rR   rU   ©rV   rW   rX   rY   rZ   r-   r[   rN   r/   r,   r¿   r¿   ±  ó   † Øó[ó ó[r/   r¿   c                  ó(   • \ rS rSr\SS j5       rSrg)ÚDoraConv2dVarianti¸  c                ó>   • [        SS9n[        R                  XUS9  g rÁ   )r   r¯   r±   rÂ   s        r,   r-   ÚDoraConv2dVariant.init¹  rÄ   r/   rN   N)
r'   r   r(   rO   r)   r   r*   r   rR   rU   rÅ   rN   r/   r,   rÈ   rÈ   ¸  rÆ   r/   rÈ   c                  ó(   • \ rS rSr\SS j5       rSrg)ÚDoraConv3dVarianti¿  c                ó>   • [        SS9n[        R                  XUS9  g rÁ   )r   r¯   r±   rÂ   s        r,   r-   ÚDoraConv3dVariant.initÀ  rÄ   r/   rN   N)
r'   r   r(   rO   r)   r   r*   r   rR   rU   rÅ   rN   r/   r,   rÌ   rÌ   ¿  rÆ   r/   rÌ   c                  ó    • \ rS rSr\S
S j5       r\SS j5       r\SS j5       r\SS j5       r\SS j5       r	\          SS j5       r
Srg	)ÚQALoraLinearVariantiÆ  c                ó6  • UR                   nU R                  b/  U R                  U-  S:w  a  [        SU R                   SU S35      eSU R                  ;  a  U R                  S-   U l        [	        U S5      (       d  0 U l         X@R                   U'   U R
                  U   nUR                  nUR                  R                  nUR                  R                  n[        R                  " UR                  U R                   U   -  USUUS	9n	X�R
                  U'   g)
a  
Initializes QALoRA specific parameters for a given adapter.

Args:
    module (Linear): The linear module to be adapted.
    adapter_name (str): The name of the adapter.
    config (LoraConfig): The config of the LoRA adapter.
    **kwargs: Additional keyword arguments.
Nr   z1`use_qalora=True` requires `module.in_features` (z)) to bedivisible by 'qalora_group_size' (Ú)Úqalora_group_size)rÓ   F)Úbiasr$   rv   )rÓ   r   r!   Úother_param_namesr%   r1   Úout_featuresr#   r$   rv   r   r   )
r'   r(   r)   r*   rÓ   Úold_lora_A_layerÚrr$   rv   Únew_lora_A_layers
             r,   r-   ÚQALoraLinearVariant.initÇ  s%  € ð #×4Ñ4ÐØ×ÑÑ)¨f×.@Ñ.@ÐCTÑ.TÐXYÓ.YÜØCÀF×DVÑDVÐCWð X5Ø5FÐ4GÀqðJóð ð
  f×&>Ñ&>Ó>Ø'-×'?Ñ'?ÐBXÑ'XˆFÔ$ä�vÐ2×3Ñ3Ø')ˆFÔ$Ø1B× Ñ  Ñ.à!Ÿ=™=¨Ñ6ÐØ×)Ñ)ˆØ!×(Ñ(×/Ñ/ˆØ ×'Ñ'×-Ñ-ˆäŸ9š9Ø×(Ñ(¨F×,DÑ,DÀ\Ñ,RÑRØØØØñ
Ðð '7�‰�lÒ#r/   c                ó   • [        S5      e)Nz;QALoRA for GPTQ layers does not support 'get_delta_weight'.©ÚNotImplementedError)r'   r8   s     r,   rw   Ú$QALoraLinearVariant.get_delta_weightî  s   € ä!Ð"_Ó`Ð`r/   c                ó   • [        S5      e)Nz5QALoRA for GPTQ layers does not support 'safe_merge'.rÜ   rC   s      r,   rE   ÚQALoraLinearVariant.merge_safeò  s   € ä!Ð"YÓZÐZr/   c                ó   • [        S5      e)Nz7QALoRA for GPTQ layers does not support 'merge_unsafe'.rÜ   rC   s      r,   rI   Ú QALoraLinearVariant.merge_unsafeö  s   € ä!Ð"[Ó\Ð\r/   c                ó   • [        S5      e)Nz2QALoRA for GPTQ layers does not support 'unmerge'.rÜ   rC   s      r,   rL   ÚQALoraLinearVariant.unmergeú  s   € ä!Ð"VÓWÐWr/   c                óØ  • U R                   U   R                  nU R                  U   R                  nU R                  U   nU R                  U   nU R
                  U   n	U R                  (       a'  [        U[        R                  5      (       d  U" U5      OUn
U
R                  n[        U5      S:”  a  U
R                  SU R                  5      nOU
nUR                  u  pÞXé-  nUR                  XßU	5      R                  SS9nUU-  nUUR                  5       -  UR                  5       -  U-  n[        U5      S:”  a'  UR                  US S UR!                  S5      4-   5      nUU-   $ )Né   ru   ©Údim)r1   r#   r2   r7   r4   rÓ   r�   rŽ   r   r�   ÚshapeÚlenr{   r   ÚmeanÚtÚsize)r'   r8   r9   r:   r*   Úlora_A_weightÚlora_B_weightr3   r4   Ú
group_sizeÚ	x_droppedÚ
orig_shapeÚx_flatÚ
batch_sizer   Úpooled_featuresÚx_pooledÚx_pooled_scaledr<   s                      r,   r=   ÚQALoraLinearVariant.forwardþ  sK  € ð Ÿ™ nÑ5×<Ñ<ˆØŸ™ nÑ5×<Ñ<ˆØ×%Ñ% nÑ5ˆØ—.‘. Ñ0ˆØ×-Ñ-¨nÑ=ˆ
à"(§/§/¼*ÀWÌbÏkÉk×:ZÑ:Z‘G˜A”JÐ`aˆ	Ø—_‘_ˆ
ô ˆz‹?˜QÓØ—^‘^ B¨×(:Ñ(:Ó;‰FàˆFà"(§,¡,Ñˆ
Ø%Ñ3ˆà—;‘;˜z¸JÓG×LÑLÐQRÐLÐSˆà" _Ñ4ˆð   -§/¡/Ó"3Ñ3°m·o±oÓ6GÑGÈ'ÑQˆô ˆz‹?˜QÓØ—J‘J˜z¨#¨2˜°%·*±*¸R³.Ð1BÑBÓCˆEà˜‰~Ðr/   rN   N©
r'   r   r(   rO   r)   r   r*   r   rR   rU   )r'   r   r8   rO   rR   rQ   rS   rT   rP   )rV   rW   rX   rY   rZ   r-   rw   rE   rI   rL   r=   r[   rN   r/   r,   rÐ   rÐ   Æ  s¯   † Øó$7ó ð$7ðL óaó ðað ó[ó ð[ð ó]ó ð]ð óXó ðXð ð$Øð$àð$ð ð$ð ð	$ð 
ó$ó ó$r/   rÐ   c                  óŒ   • \ rS rSr\S	S j5       r\S
S j5       r\SS j5       r\S
S j5       r\          SS j5       r	Sr
g)ÚALoraLinearVarianti&  c                ó   • g ©NrN   )r'   r(   r)   r*   s       r,   r-   ÚALoraLinearVariant.init'  s   € àr/   c                ó   • [        S5      e)Nz$aLoRA does not support safe merging.rÜ   rC   s      r,   rE   ÚALoraLinearVariant.merge_safe+  s   € ä!Ð"HÓIÐIr/   c                ó   • [        S5      e)NzaLoRA does not support merging.rÜ   rC   s      r,   rI   ÚALoraLinearVariant.merge_unsafe/  s   € ä!Ð"CÓDÐDr/   c                ó   • [        S5      e)Nz!aLoRA does not support unmerging.rÜ   rC   s      r,   rL   ÚALoraLinearVariant.unmerge3  s   € ä!Ð"EÓFÐFr/   c                ó˜  • UR                  SS 5      nU R                  U   nU R                  U   nU R                  U   nU R                  U   n	UR                  UR                  R                  5      nUR                  n
U
S   n[        U
5      S:X  a  U
S   nOSnU
S   nUR                  S   nUR                  nUc%  [        R                  " X¼4[        R                  S9nO‡[        R                  " U Vs/ s H  nUc  SO[        [!        U5      U5      PM     snU[        R"                  S9n[        R$                  " XÏS9R'                  S5      nUUU-
  R'                  S5      :¬  nUR)                  SU5      nUR)                  SU5      nUR)                  S5      nUU==   U" U" U" UU   5      5      5      U	-  -  ss'   U$ s  snf )	NÚalora_offsetsr   é   r
   ru   ©rv   )r$   rv   )r$   )Úgetr1   r2   r7   r4   r"   r#   rv   ré   rê   r$   ÚtorchÚzerosÚboolÚtensorÚminÚintÚlongÚarangeÚ	unsqueezer{   )r'   r8   r9   r:   r*   r  r1   r2   r3   r4   Úresult_shapeÚBrž   ÚDÚDxr$   ÚmaskÚoÚoffsetsÚposró   Úres_flatÚ	mask_flats                          r,   r=   ÚALoraLinearVariant.forward7  s¦  € ð Ÿ
™
 ?°DÓ9ˆØ—‘˜~Ñ.ˆØ—‘˜~Ñ.ˆØ×%Ñ% nÑ5ˆØ—.‘. Ñ0ˆØ�D‰D�—‘×$Ñ$Ó%ˆØ—|‘|ˆØ˜‰OˆÜˆ|Ó Ó!Ø˜Q‘‰AàˆAØ˜ÑˆØ�W‰W�R‰[ˆØ—‘ˆØÑ Ü—;’; ˜v¬U¯Z©ZÑ8‰Dô
 —l’lÙ=JÓKº]¸�a‘i‘¤S¬¨Q«°£^Ò3¹]ÑKØÜ—j‘jñˆGô —,’,˜qÑ0×:Ñ:¸1Ó=ˆCØ˜1˜w™;×1Ñ1°!Ó4Ñ4ˆDð —‘˜˜B“ˆØ—;‘;˜r 1Ó%ˆØ—I‘I˜b“Mˆ	ð 	�Ó™v¡f©W°V¸IÑ5FÓ-GÓ&HÓIÈGÑSÑSÓØˆùò Ls   Ã8$GrN   Nrù   rS   rT   rP   r”   rN   r/   r,   rû   rû   &  s˜   † Øóó ðð óJó ðJð óEó ðEð óGó ðGð ð,Øð,àð,ð ð,ð ð	,ð 
ó,ó ó,r/   rû   c           	     óŒ  • Uc  / $ UR                   S   nS/U-  n0 n[        R                  " [        5      n[	        U5       Hµ  nU(       a  U[        U5      :  a  X8   OUn	U	S:X  a  SXX'   M+  X�;  a#  [        R                  " SU	 SU S35        SXX'   MS  X	   n
[        U
SS5      nUc  SXX'   Mm  X–;  a0  [        R                  " U[        R                  UR                  S9Xi'   Xy   R                  U5        M·     UR                  5        Hµ  u  pÍXl   n[        U5      nU H›  nX(   n[        U5      nS	nUUS   :H  R                  S
S9S   nU HL  nUR!                  5       nUU-   U::  d  M  [        R"                  " UUUU-    U5      (       d  MB  UU:”  d  MJ  UnMN     US	:w  a  UU-
  nUS:”  a  UOSXX'   M—  SXX'   M�     M·     U$ )a^  
This is a helper function for Activated LoRA (aLoRA) that searches each input token sequence for the last
occurrence of the appropriate "alora_invocation_tokens" invocation sequence. The calculated alora_offset is the
location of the *start* of the invocation tokens, counting backward from the end (will therefore always be >=
len(alora_invocation_tokens). If adapter_names is passed, then each input uses the appropriate invocation sequence
for the specified adapter for that row. Logic is provided to handle mixed collections of adapters for which not all
are aLoRAs (e.g. some base model, some LoRA).
Nr   Ú__base__z	Adapter 'z5' not found in peft_config. Using base model for row Ú.Úalora_invocation_tokens©rv   r$   ru   T)Úas_tuple)ré   ÚcollectionsÚdefaultdictÚlistÚrangerê   ÚwarningsÚwarnrj   r
  r  r  r$   ÚappendÚitemsÚnonzeroÚitemÚequal)Úpeft_configr8   Ú	input_idsÚadapter_namesrô   r  Úcached_invocation_tensorsÚadapters_to_process_indicesÚiÚcurrent_adapter_nameÚcurrent_peft_configÚinvocation_tokensÚadapter_name_to_processÚindicesÚcurrent_invocation_ids_tensorÚinvocation_lenÚsequenceÚseq_lenÚbest_match_start_idxÚpossible_startsÚstart_idx_tensorÚidxÚ
offset_vals                          r,   Úcalculate_alora_offsetsrC  g  s  € ð ÑØˆ	à—‘ Ñ#€JØ�F˜ZÑ'€Mà "ÐÜ"-×"9Ò"9¼$Ó"?Ðä�:ÖˆÞ3@ÀQÌÈ]ÓI[ÓE[˜}Ò/ÐaoÐà :Ó-Ø#ˆMÑÙàÓ2Ü�MŠM˜IÐ&:Ð%;Ð;pÐqrÐpsÐstÐuÔvØ#ˆMÑÙà)Ñ?Ðä#Ð$7Ð9RÐTXÓYÐØÑ$Ø#ˆMÑÙàÓ@Ü>C¿lºlØ!¬¯©¸I×<LÑ<Lñ?Ð%Ñ;ð 	$Ñ9×@Ñ@ÀÖCñ1 ð4 -H×,MÑ,MÖ,OÑ(ÐØ(AÑ(ZÐ%ÜÐ:Ó;ˆãˆAØ ‘|ˆHÜ˜(“mˆGØ#%Ð à'Ð+HÈÑ+KÑK×TÑTÐ^bÐTÐcÐdeÑfˆOã$3Ð Ø&×+Ñ+Ó-�Ø˜Ñ'¨7Õ2Ü—{’{ 8¨C°#¸Ñ2FÐ#GÐIf×gÓgØÐ!5Õ5Ø36Ò0ñ %4ð $ rÓ)Ø$Ð';Ñ;�
Ø1;¸a³¡:ÀT�Ó à#'�Ó ó% ñ	 -Pð. Ðr/   c                óð   • Sn[        U R                  SS5      (       a  SnU$ U(       aL  U HF  nUS:X  a  M  U R                  R                  U5      nU(       d  M/  [        USS5      (       d  MC  Sn  U$    U$ )zK
Helper function to determine if the current batch has any aLoRA adapters.
Fr!  NTr  )rj   Úactive_peft_configr/  r	  )Úmodelr1  Úis_alora_relevantÚnameÚconfig_s        r,   Úis_alora_relevant_in_batchrJ  ¯  s‰   € ð ÐÜˆu×'Ñ'Ð)BÀD×IÑIØ Ðð Ðö 
Û!ˆDØ�zÓ!ÙØ×'Ñ'×+Ñ+¨DÓ1ˆGßˆwœ7 7Ð,EÀt×LÓLØ$(Ð!ØàÐñ "ð Ðr/   c                ó  • UR                  SS5      n[        X5      (       d  U$ UR                  S5      nUcP  Uc   Ub  [        R                  " S5        SUS'   U$ Ub%  [	        U R
                  U R                  UUS9US'   U$ SUS'   U$ )z�
Wrapper around calculate_alora_offsets, for the .forward of the model. It only calculates alora_offsets if the
batch contains aLoRA adapters.
r1  Nr  zkCannot calculate aLoRA offsets when only inputs_embeds are provided. Disabling aLoRA for this forward pass.©r1  )r	  rJ  r(  r)  rC  r/  r8   )rF  r0  Úinputs_embedsr*   Úadapter_names_for_offset_calcr  s         r,   Úget_alora_offsets_for_forwardrO  Â  s­   € ð %+§J¡J¨ÀÓ$EÐ!Ü% e×KÑKàˆØ—J‘J˜Ó/€MØÑØÑ Ñ!:Ü�MŠMØ}ôð '+ˆF�?Ñ#ð €Mð Ñ"Ü&=Ø×!Ñ!Ø×$Ñ$ØØ;ñ	'ˆF�?Ñ#ð €Mð '+ˆF�?Ñ#Ø€Mr/   c                óÈ  • UR                  S5      n[        X5      (       d  U$ UR                  S5      nUcª  UR                  S5      nUc1  U(       a(  [        US   [        R                  5      (       a  US   nOSnUbG  UR
                  S:X  a  UR                  S5      n[        U R                  U R                  UUS9nXbS'   U$ [        R                  " S5        SUS'   U$ )	z�
Wrapper around calculate_alora_offsets, for the .generate of the model. It only calculates alora_offsets if the
batch contains aLoRA adapters.
r1  r  Nr0  r   r
   rL  z_Cannot calculate aLoRA offsets during generate as input_ids are not available. Disabling aLoRA.)r	  rJ  rŽ   r
  ÚTensorÚndimr  rC  r/  r8   r(  r)  )rF  Úargsr*   rN  Úalora_offsets_from_kwargsÚcurrent_input_idsÚcalculated_offsetss          r,   Úget_alora_offsets_for_generaterW  à  sï   € ð
 %+§J¡J¨Ó$?Ð!Ü% e×KÑKàˆØ &§
¡
¨?Ó ;ÐØ Ñ(Ø"ŸJ™J {Ó3ÐØÑ$Þœ
 4¨¡7¬E¯L©L×9Ñ9Ø$(¨¡GÑ!à$(Ð!àÑ(Ø ×%Ñ%¨Ó*Ø$5×$?Ñ$?ÀÓ$BÐ!Ü!8Ø×!Ñ!Ø×$Ñ$Ø!Ø;ñ	"Ðð '9�?Ñ#ð €Mô �MŠMØqôð '+ˆF�?Ñ#Ø€Mr/   c                  ó�   ^ • \ rS rSrS\R
                  \R                  " S5      4           SU 4S jjjrS	S jrS r	Sr
U =r$ )
ÚBlockDiagonalLineari  Frd   c           	     ó4  >• [         TU ]  5         Xl        X l        X0l        U R                  U-  S:w  d  U R                  U-  S:w  a2  [        SU R                   SU R                   SU R                   35      e[        R                  " [        R                  " X!U-  XVS95      U l
        U(       a4  [        R                  R                  R                  U R                  5        g [        R                  R                  R                  U R                  5        g )Nr   zself.in_features=z or self.out_features=z not divisible by r"  )ÚsuperÚ__init__r   rÖ   Únblocksr!   r   Ú	Parameterr
  Úemptyr#   r-   Úzeros_Úkaiming_uniform_)Úselfr   rÖ   r]  Ú	init_zerorv   r$   Ú	__class__s          €r,   r\  ÚBlockDiagonalLinear.__init__  sÛ   ø€ ô 	‰ÑÔØ&ÔØ(ÔØŒØ×Ñ˜gÑ%¨Ó*¨d×.?Ñ.?À'Ñ.IÈQÓ.NÜØ# D×$4Ñ$4Ð#5Ð5KÈD×L]ÑL]ÐK^Ð^pÐqu×q}Ñq}Ðp~Ðóð ô —l’l¤5§;¢;¨|ÈGÑ=SÐ[`Ñ#pÓqˆŒæÜ�H‰H�M‰M× Ñ  §¡Õ-ä�H‰H�M‰M×*Ñ*¨4¯;©;Õ7r/   c                ó°  • UR                   S S nUR                  5       S:w  a  UR                  SUR                   S   5      nUR                   S   nU R                  nUR                   S   U-  nU R                  U-  nUR                  X4U5      nU R
                  R                  XFU5      n[        R                  " SX5      nUR                  " / UQSP76 $ )Nru   ræ   r   zbim,inm->bin)	ré   rè   Úreshaper]  rÖ   r#   r{   r
  Úeinsum)	rb  r9   Ú
first_dimsr  ÚnbÚmÚnÚwÚouts	            r,   r=   ÚBlockDiagonalLinear.forward   s»   € Ø—W‘W˜S˜b�\ˆ
Ø�5‰5‹7�a‹<Ø—	‘	˜"˜aŸg™g b™kÓ*ˆAØ�G‰G�A‰JˆØ�\‰\ˆØ�G‰G�B‰K˜2ÑˆØ×Ñ Ñ#ˆØ�I‰I�a˜QÓˆØ�K‰K×Ñ˜R AÓ&ˆÜ�lŠl˜>¨1Ó0ˆØ�{Š{Ð+˜JÐ+¨Ò+Ð+r/   c                óx   • [         R                  " [         R                  " U R                  U R                  SS96 $ )z˜Returns weight in a format similar to a vanilla LoRA adapter. For this, we stack the blocks on the diagonal,
leaving the off-diagonals padded with zero.r   rç   )r
  Ú
block_diagÚchunkr#   r]  )rb  s    r,   Úweight_as_blockdiagonal_matrixÚ2BlockDiagonalLinear.weight_as_blockdiagonal_matrix-  s*   € ô ×Ò¤§¢¨T¯[©[¸$¿,¹,ÈAÑ!NÐOÐOr/   )r   r]  rÖ   r#   )r   r  rÖ   r  r]  r  rc  r  rv   ztorch.dtyper$   ztorch.device)r9   rQ   rR   rQ   )rV   rW   rX   rY   r
  Úfloat32r$   r\  r=   rs  r[   Ú__classcell__)rd  s   @r,   rY  rY    sp   ø† ð  Ø"Ÿ]™]Ø$Ÿ|š|¨EÓ2ð8àð8ð ð8ð ð	8ð
 ð8ð ð8ð ÷8ð 8ô2,÷Pð Pr/   rY  c                  ó    • \ rS rSr\SS j5       r\SS j5       r\SS j5       r\SS j5       r\SS j5       r	\SS j5       r
\SS j5       rS	rg
)ÚBdLoraLinearVarianti3  c           	     óÔ  ^• UR                   nUR                  SS5      m[        U[        5      (       a  [	        S0 UD6nUR
                  =(       d    / nUR                  =(       d    / nUR                  n[        U4S jU 5       5      n[        U4S jU 5       5      n	U(       a  U	(       a  [        ST S35      eUR                  (       a  U(       d  U	(       d  [        ST S35      eU(       ax  U R                  U   R                  n
U R                  5       n[        UR                  U
USUR                   R"                  UR                   R$                  S	9nXÀR                  U'   g U	(       ax  U R&                  U   R                  n
U R                  5       n[        U
UR                  US
UR                   R"                  UR                   R$                  S	9nXÀR&                  U'   g g )NÚtarget_nameÚ c              3  ó,   >#   • U  H	  oT;   v •  M     g 7frý   rN   ©Ú.0Úpatternrz  s     €r,   Ú	<genexpr>Ú+BdLoraLinearVariant.init.<locals>.<genexpr>A  ó   øé € Ð&jÒMiÀ'°+Ö'=ÒMiùó   ƒc              3  ó,   >#   • U  H	  oT;   v •  M     g 7frý   rN   r}  s     €r,   r€  r�  B  r‚  rƒ  zTarget z- matches both A and B block-diagonal patternsz‡ matches neither A nor B block-diagonal patterns.If this is intentional, set match_strict=False in BdLoraConfig during initialization. F)r]  rc  rv   r$   TrN   )Ú
use_bdlorar	  rŽ   Údictr   Útarget_modules_bd_aÚtarget_modules_bd_br]  Úanyr!   Úmatch_strictr1   rÖ   rn   rY  r   r#   rv   r$   r2   )r'   r(   r)   r*   r…  Úlora_a_blockdiagonal_patternÚlora_b_blockdiagonal_patternr]  Úhas_lora_a_blockdiagonalÚhas_lora_b_blockdiagonalrØ   rg   Úlayerrz  s                @r,   r-   ÚBdLoraLinearVariant.init4  s¶  ø€ à×&Ñ&ˆ
Ø—j‘j °Ó3ˆô �j¤$×'Ñ'Ü%Ñ3¨
Ñ3ˆJà'1×'EÑ'E×'KÈÐ$Ø'1×'EÑ'E×'KÈÐ$Ø×$Ñ$ˆä#&Ô&jÑMiÓ&jÓ#jÐ Ü#&Ô&jÑMiÓ&jÓ#jÐ æ#Ö(@Ü˜w { mÐ3`ÐaÓbÐbØ×"×"Ö,DÖH`ÜØ˜+˜ð 'ið ióð ö
 $Ø—‘˜lÑ+×8Ñ8ˆAØ×.Ñ.Ó0ˆJÜ'Ø×&Ñ&ØØØØ ×'Ñ'×-Ñ-Ø!×(Ñ(×/Ñ/ñˆEð +0�M‰M˜,Ò'Þ%Ø—‘˜lÑ+×7Ñ7ˆAØ×.Ñ.Ó0ˆJÜ'ØØ×'Ñ'ØØØ ×'Ñ'×-Ñ-Ø!×(Ñ(×/Ñ/ñˆEð +0�M‰M˜,Ò'ð &r/   c                óþ   • U R                   U   nU R                  U   nU R                  U   nU R                  U   nU" U5      nU R	                  X%R
                  R                  5      nX6" U" U5      5      U-  -  nU$ rý   )r1   r2   r7   r4   Ú_cast_input_dtyper#   rv   )	r'   r8   r9   r:   r*   r1   r2   r3   r4   s	            r,   r=   ÚBdLoraLinearVariant.forwarde  s|   € à—‘˜~Ñ.ˆØ—‘˜~Ñ.ˆØ×%Ñ% nÑ5ˆØ—.‘. Ñ0ˆÙ�A‹Jˆà×$Ñ$ Q¯©×(;Ñ(;Ó<ˆØ�&™ ›Ó# gÑ-Ñ-ˆØˆr/   c                ód   • [        U [        5      (       a  U R                  5       $ U R                  $ rý   )rŽ   rY  rs  r#   )r'   s    r,   Ú+_get_weight_from_module_maybe_blockdiagonalÚ?BdLoraLinearVariant._get_weight_from_module_maybe_blockdiagonalq  s)   € ä�fÔ1×2Ñ2Ø×8Ñ8Ó:Ð:à—=‘=Ð r/   c                óŠ  • U R                   U   R                  R                  nU R                  5       nUR                  R                  nUR
                  S:H  =(       a-    U[        R                  :H  =(       d    U[        R                  :H  n[        R                  U R                  U   5      n[        R                  U R                   U   5      nU(       a   UR                  5       nUR                  5       n[        Xv-  U R                  5      U R                  U   -  nU(       a  UR!                  US9nUR!                  US9$ )zmSimilar to get_delta_weight for a linear module, but we have to eventually reshape the blocks
of the weights.rd   r  )r2   r#   r$   rn   rv   rl   r
  Úfloat16Úbfloat16rx  r•  r1   Úfloatr	   rb   r4   r"   )	r'   Úadapterr$   rg   rv   Úcast_to_fp32Úweight_AÚweight_BÚoutput_tensors	            r,   Ú_get_bdlora_delta_weightÚ,BdLoraLinearVariant._get_bdlora_delta_weightx  s  € ð —‘˜wÑ'×.Ñ.×5Ñ5ˆà×*Ñ*Ó,ˆ
Ø×!Ñ!×'Ñ'ˆØ—{‘{ eÑ+×c°¼%¿-¹-Ñ1G×1bÈ5ÔTY×TbÑTbÑKbˆä&×RÑRÐSY×S`ÑS`ÐahÑSiÓjˆÜ&×RÑRÐSY×S`ÑS`ÐahÑSiÓjˆæØ—~‘~Ó'ˆHØ—~‘~Ó'ˆHä! (Ñ"5°v×7LÑ7LÓMÐPV×P^ÑP^Ð_fÑPgÑgˆæØ)×,Ñ,°5Ð,Ð9ˆMð ×Ñ eÐÐ,Ð,r/   c                ó2   • U[         R                  X5      -   $ rý   ©rx  r   rC   s      r,   rE   ÚBdLoraLinearVariant.merge_safe‘  ó   € àÔ0×IÑIÈ&ÓaÑaÐar/   c                óT   • U=R                   [        R                  X5      -  sl         g rý   )r„   rx  r   rC   s      r,   rI   Ú BdLoraLinearVariant.merge_unsafe•  s   € à×ÒÔ/×HÑHÈÓ`Ñ`Ör/   c                ó2   • U[         R                  X5      -
  $ rý   r£  rC   s      r,   rL   ÚBdLoraLinearVariant.unmerge™  r¥  r/   rN   N)r'   r   r(   rO   r)   r   rR   rU   rP   )r'   r½   rR   rQ   )r'   r   r›  rO   rR   rQ   rS   rT   )rV   rW   rX   rY   rZ   r-   r=   r•  r   rE   rI   rL   r[   rN   r/   r,   rx  rx  3  s“   † Øó.0ó ð.0ð` ó	ó ð	ð ó!ó ð!ð ó-ó ð-ð0 óbó ðbð óaó ðað óbó óbr/   rx  rý   )
r/  r   r8   rO   r0  rQ   r1  úOptional[list[str]]rR   z	list[int])rF  r½   r1  rª  )NN)rF  r½   r0  rQ   rM  rQ   )rF  z	nn.module)2Ú
__future__r   r$  r(  Útypingr   r   r
  Úaccelerate.utils.importsr   r   Úpeft.tuners.lora.configr   Úpeft.utils.otherr	   r;   r   r)   r   r   Údorar   r   r   r   r   r�  r   r   r   r   r   r   r   r   r]   r–   r¯   r¿   rÈ   rÌ   rÐ   rû   rC  rJ  rO  rW  ÚModulerY  rx  rN   r/   r,   Ú<module>r²     sL  ðõ #ã Û ß  ã Ý 5Ý å 0Ý &å 'ß *ß hÕ hß R× RÑ Rôa^˜ô a^ôHm˜ô mô``Ð,ô `ôFV˜ô Vôr[Ð*ô [ô[Ð*ô [ô[Ð*ô [ô]˜+ô ]ô@>˜ô >ðD quðEØðEØ-0ðEØ=IðEØZmðEàõEöPð( UYðØðØ!-ðØEQõô<#ôL*P˜"Ÿ)™)ô *PôZhb˜+õ hbr/   