ó
    >:j &  ã                  óº   • S SK Jr  S SKJr  S SKJrJr  S SKrS SKJ	r	  S SKJ
r
  S SKJr  SSKJr   " S	 S
5      r " S S\5      r " S S\5      r " S S\5      rg)é    )Úannotations)ÚCallable)ÚAnyÚOptionalN)ÚTensor)Úloraé   )ÚXLoraConfigc                  ó\   • \ rS rSrSr            SS jr \S	S j5       r S
S jrSr	g)Ú
XLoraLayeré   zÂ
A XLoraLayer wraps any LoraLayer and performs the XLora operation on the LoRA adaptors specified. Its primary API
is the forward method, which uses the scalings to execute the XLora algorithm.
c                ó@   • Xl         X0l        X l        X@l        XPl        g ©N)ÚmodelÚtarget_forwardÚtargetÚlayer_numberÚconfig)Úselfr   r   r   r   r   s         ÚT/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/xlora/layer.pyÚ__init__ÚXLoraLayer.__init__"   s   € ð Œ
Ø,ÔØŒØ(ÔØ�ó    c                ó@   • US S 2S S 2U4   R                  S5      nX-  $ )Néÿÿÿÿ)Ú	unsqueeze)ÚxÚscalings_layerÚadapterÚscalingss       r   Úapply_scalings_to_xÚXLoraLayer.apply_scalings_to_x4   s'   € ð "¢!¢Q¨ -Ñ0×:Ñ:¸2Ó>ˆà‰|Ðr   c                ó*  • US S 2S S 2U R                   S S 24   nU R                  R                  b~  [        R                  " X R                  R                  SS9u  p4[        R
                  " U[        R                  S9nUR                  SUS5        X%R                  UR                  5      -  nU R                  R                  (       aI  US:g  nUR                  U) [        S5      5      n[        R                  " USS9nUR                  U) S5      nU$ )	Nr   )ÚkÚdim)ÚdtypeTr   z-inf)r%   g        )r   r   Ú
top_k_loraÚtorchÚtopkÚ
zeros_likeÚboolÚscatter_Útor&   Úenable_softmax_topkÚmasked_fillÚfloatÚsoftmax)	r   r    Úxlora_scalingsÚ_Útopk_indicesÚmaskÚnonzero_maskÚfullÚnew_scalingss	            r   Úget_maybe_topk_scalingsÚ"XLoraLayer.get_maybe_topk_scalings@   sâ   € à!)ª!ªQ°×0AÑ0AÂ1Ð*DÑ!Eˆà�;‰;×!Ñ!Ñ-Ü#Ÿjšj¨¿;¹;×;QÑ;QÐWYÑZ‰OˆAô ×#Ò# N¼%¿*¹*ÑEˆDØ�M‰M˜"˜l¨DÔ1à+¯g©g°n×6JÑ6JÓ.KÑKˆNð �;‰;×*×*Ø)¨QÑ.ˆLØ!×-Ñ-¨|¨m¼UÀ6»]ÓKˆDÜ Ÿ=š=¨°2Ñ6ˆLØ)×5Ñ5°|°mÀSÓIˆNàÐr   )r   r   r   r   r   N)r   ú	nn.Moduler   zlora.LoraLayerr   úCallable[..., Any]r   Úintr   r
   ÚreturnÚNone)r   útorch.Tensorr   r@   r   r=   r>   r@   )r>   r@   )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   Ústaticmethodr!   r9   Ú__static_attributes__© r   r   r   r      si   † ñð
àðð ðð +ð	ð
 ðð ðð 
ôðð óó ðð÷
r   r   c                  óV   ^ • \ rS rSr            SU 4S jjrSS.SS jjrSrU =r$ )	ÚXLoraLinearLayeréW   c                ó(   >• [         TU ]  XX4U5        g r   ©Úsuperr   ©r   r   r   r   r   r   Ú	__class__s         €r   r   ÚXLoraLinearLayer.__init__X   ó   ø€ ô 	‰Ñ˜¨ÀfÕMr   N©r    c          	     ó„  • UR                   nUb  U R                  U5      nU R                  R                  " U/UQ70 UD6nU R                  R                  (       GdR  [        U R                  R                  5       GH.  u  p‰X�R                  R                  R                  5       ;  a  M/  U R                  R                  U	   (       a  [        S5      eU R                  R                  U	   n
U R                  R                  U	   nU R                  R                  U	   nU R                  R                  U	   nUR                  U
R                  R                   5      nUb*  U R!                  UWU5      nU R"                  R$                  nOUnSnX{" U
" U" U5      5      5      U-  U-  -  nGM1     UR                  U5      nU$ ©úµ
This method is designed to be a drop-in-replacement for the LoRA layers' .forward method. To use it, a bound
method must be created (bound to an instance of the XLoraLayer class).
ú7X-LoRA currently does not support LoRA layers with DoRAr	   ©r&   r9   r   Ú
base_layerÚmergedÚ	enumerateÚactive_adaptersÚlora_AÚkeysÚuse_doraÚ
ValueErrorÚlora_BÚlora_dropoutÚscalingr-   Úweightr!   r   Úglobal_scaling_weight©r   r   r    ÚargsÚkwargsÚprevious_dtyper2   ÚresultÚ	adapter_nÚactive_adapterr]   ra   Údropoutrc   Úx_modÚscaling_weights                   r   ÚforwardÚXLoraLinearLayer.forwardb   s|  € ð Ÿ™ˆØÑØ!×9Ñ9¸(ÓCˆNà—‘×'Ò'¨Ð;¨DÒ;°FÑ;ˆð �{‰{×!×!Ð!Ü-6°t·{±{×7RÑ7R×-SÑ)�	Ø!¯©×);Ñ);×)@Ñ)@Ó)BÓBÙà—;‘;×'Ñ'¨×7Ü$Ð%^Ó_Ð_ØŸ™×+Ñ+¨NÑ;�ØŸ™×+Ñ+¨NÑ;�ØŸ+™+×2Ñ2°>ÑB�ØŸ+™+×-Ñ-¨nÑ=�Ø—D‘D˜Ÿ™×,Ñ,Ó-�ØÑ'Ø ×4Ñ4°Q¸È	ÓR�EØ%)§[¡[×%FÑ%F‘Nà�EØ%&�NØ˜&¡©°«Ó!7Ó8¸7ÑBÀ^ÑSÑS“ñ# .Tð& —‘˜>Ó*ˆØˆr   rH   )r   r;   r   zlora.Linearr   r<   r   r=   r   r
   r>   r?   ©
r   r   rg   r   r    zOptional[Tensor]rh   r   r>   r   ©rA   rB   rC   rD   r   rp   rG   Ú__classcell__©rP   s   @r   rJ   rJ   W   sZ   ø† ðNàðNð ðNð +ð	Nð
 ðNð ðNð 
÷Nð KO÷ "õ "r   rJ   c                  óV   ^ • \ rS rSr            SU 4S jjrSS.SS jjrSrU =r$ )	ÚXLoraEmbeddingLayeré‡   c                ó(   >• [         TU ]  XX4U5        g r   rM   rO   s         €r   r   ÚXLoraEmbeddingLayer.__init__ˆ   rR   r   NrS   c               ó†  • Ub  U R                  U5      nU R                  R                  " U/UQ70 UD6nU R                  R                  5       nU R                  R                  (       GdV  [        U R                  R                  5       GH2  u  p‰X�R                  R                  ;  a  M!  U R                  R                  R                  U	S5      (       a  [        S5      eU R                  R                  U	   R                  n
U R                  R                  U	   R                  nU R                  R                  U	   nU R                  R                  X5      nUb*  U R                  UWU5      nU R                   R"                  nOUnSnXë-  U-  U-  nUb  UUR%                  UR&                  5      -  nUU-  nGM5     U$ )rV   FrW   r	   )r9   r   rY   Ú_get_embed_scalerZ   r[   r\   Úlora_embedding_Ar_   Úgetr`   ÚTÚlora_embedding_Brc   Ú_embedr!   r   re   r-   r&   )r   r   r    rg   rh   r2   rj   Úembed_scalerk   rl   Úembedding_AÚembedding_Brc   Úafter_AÚafter_A_modro   Úadapter_outputs                    r   rp   ÚXLoraEmbeddingLayer.forward’   s‹  € ð ÑØ!×9Ñ9¸(ÓCˆNà—‘×'Ò'¨Ð;¨DÒ;°FÑ;ˆð —k‘k×2Ñ2Ó4ˆð �{‰{×!×!Ð!Ü-6°t·{±{×7RÑ7R×-SÑ)�	Ø!¯©×)EÑ)EÓEÙà—;‘;×'Ñ'×+Ñ+¨N¸E×BÑBÜ$Ð%^Ó_Ð_Ø"Ÿk™k×:Ñ:¸>ÑJ×LÑL�Ø"Ÿk™k×:Ñ:¸>ÑJ×LÑL�ØŸ+™+×-Ñ-¨nÑ=�ØŸ+™+×,Ñ,¨QÓ<�ØÑ'Ø"&×":Ñ":¸7ÀNÐT]Ó"^�KØ%)§[¡[×%FÑ%F‘Nà")�KØ%&�Nà"-Ñ";¸wÑ!FÈÑ!W�ð Ñ*Ø%3°k·n±nÀ^×EYÑEYÓ6ZÑ%Z�Nà˜.Ñ(“ñ/ .Tð2 ˆr   rH   )r   r;   r   zlora.Embeddingr   r<   r   r=   r   r
   r>   r?   rr   rs   ru   s   @r   rw   rw   ‡   sZ   ø† ðNàðNð ðNð +ð	Nð
 ðNð ðNð 
÷Nð KO÷ *õ *r   rw   c                  óV   ^ • \ rS rSr            SU 4S jjrSS.SS jjrSrU =r$ )	ÚXLoraConv2dLayeré¿   c                ó(   >• [         TU ]  XX4U5        g r   rM   rO   s         €r   r   ÚXLoraConv2dLayer.__init__À   rR   r   NrS   c          	     ó„  • UR                   nUb  U R                  U5      nU R                  R                  " U/UQ70 UD6nU R                  R                  (       GdR  [        U R                  R                  5       GH.  u  p‰X�R                  R                  R                  5       ;  a  M/  U R                  R                  U	   (       a  [        S5      eU R                  R                  U	   n
U R                  R                  U	   nU R                  R                  U	   nU R                  R                  U	   nUR                  U
R                  R                   5      nUb*  U R!                  UWU5      nU R"                  R$                  nOUnSnX{" U
" U" U5      5      5      U-  U-  -  nGM1     UR                  U5      nU$ rU   rX   rf   s                   r   rp   ÚXLoraConv2dLayer.forwardÊ   s|  € ð Ÿ™ˆàÑØ!×9Ñ9¸(ÓCˆNà—‘×'Ò'¨Ð;¨DÒ;°FÑ;ˆð �{‰{×!×!Ð!Ü-6°t·{±{×7RÑ7R×-SÑ)�	Ø!¯©×);Ñ);×)@Ñ)@Ó)BÓBÙà—;‘;×'Ñ'¨×7Ü$Ð%^Ó_Ð_ØŸ™×+Ñ+¨NÑ;�ØŸ™×+Ñ+¨NÑ;�ØŸ+™+×2Ñ2°>ÑB�ØŸ+™+×-Ñ-¨nÑ=�Ø—D‘D˜Ÿ™×,Ñ,Ó-�ØÑ'Ø ×4Ñ4°Q¸È	ÓR�EØ%)§[¡[×%FÑ%F‘Nà�EØ%&�NØ˜&¡©°«Ó!7Ó8¸7ÑBÀ^ÑSÑS“ñ# .Tð& —‘˜>Ó*ˆØˆr   rH   )r   r;   r   zlora.Conv2dr   r<   r   r=   r   r
   r>   r?   rr   rs   ru   s   @r   rŠ   rŠ   ¿   sZ   ø† ðNàðNð ðNð +ð	Nð
 ðNð ðNð 
÷Nð KO÷ #õ #r   rŠ   )Ú
__future__r   Úcollections.abcr   Útypingr   r   r(   Útorch.nnÚnnr   Úpeft.tunersr   r   r
   r   rJ   rw   rŠ   rH   r   r   Ú<module>r–      sP   ðõ #å $ß  ã Ý Ý å å ÷8ñ 8ôv-�zô -ô`5˜*ô 5ôp.�zõ .r   