ó
    >:jÙ%  ã                   ó¤   • S SK r S SKJrJr  S SKrS SKJr  S SKJs  Jr	  SSK
Jr   " S S\R                  5      r " S S\5      r " S	 S
\5      rg)é    N)ÚOptionalÚUnioné   )ÚTRANSFORMERS_MODEL_CONFIGc                   óR   ^ • \ rS rSrSr\R                  4S\S\4U 4S jjjr	Sr
U =r$ )Ú_BaseAdaptedAttentioné   zEBase module, which defines adaption prompts for multiple model types.Ú
model_typeÚadapter_lenc           
      ó,  >• [        U[        5      (       a  [        S5      e[        TU ]  5         Xl        X0l        X l        [        UR                  5       5      R                  nU R                  R                  R                  nU R                  R                  R                  U l        [        R                   " ["        R$                  " SX&XTS9R'                  5       5      U l        [        R                   " ["        R*                  " SXTS95      U l        g)a  
Initialize object.

Args:
    model_type: The transformer model type. This is used to retrieve the right method to
        compute query states.
    adapter_len: The length of the adaption prompt to insert.
    model: The original transformer attention module that is being wrapped.
z)Unable to stack multiple adaption promptsr   )ÚdeviceÚdtypeN)Ú
isinstancer   Ú
ValueErrorÚsuperÚ__init__r
   Úmodelr   ÚnextÚ
parametersr   ÚconfigÚhidden_sizeÚnum_attention_headsÚ	num_headsÚnnÚ	ParameterÚtorchÚemptyÚnormal_Úadaption_promptÚzerosÚadaption_gate)Úselfr
   r   r   Útarget_dtyper   r   Ú	__class__s          €Ú^/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/adaption_prompt/layer.pyr   Ú_BaseAdaptedAttention.__init__   sË   ø€ ô �eÔ2×3Ñ3ÜÐHÓIÐIÜ‰ÑÔØ$ŒØŒ
Ø&Ôô �e×&Ñ&Ó(Ó)×0Ñ0ˆð —j‘j×'Ñ'×3Ñ3ˆØŸ™×*Ñ*×>Ñ>ˆŒä!Ÿ|š|Ü�KŠK˜˜;¸FÑW×_Ñ_Óaó 
ˆÔô  Ÿ\š\¬%¯+ª+°aÀÑ*[Ó\ˆÕó    )r   r!   r   r   r
   r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   Úfloat32ÚstrÚintr   Ú__static_attributes__Ú__classcell__©r$   s   @r%   r   r      s)   ø† ÙOàNSÏmÉmñ ] 3ð ]°S÷ ]ö ]r'   r   c                   ób  ^ • \ rS rSrSrU 4S jr       SS\\\R                        S\\\R                        S\\R                     S\\R                     S\\R                     S	\\R                     S
\\   S\\   S\\\R                  \\R                     4   S4   4S jjrSrU =r$ )ÚAdaptedAttentionGPTé>   zEThis module wraps a GPT2Attention module and injects adaption promptsc                 ó  >• UR                   R                  R                  [        R                  [        R
                  4;  a   UR                   R                  R                  O[        R                  n[        TU ]!  XX4S9  g ©N)r#   )	Úc_projÚweightr   r   Úint8Úuint8r-   r   r   ©r"   r
   r   r   r#   r$   s        €r%   r   ÚAdaptedAttentionGPT.__init__A   ó_   ø€ à).¯©×)<Ñ)<×)BÑ)BÌ5Ï:É:ÔW\×WbÑWbÐJcÓ)cˆE�L‰L×Ñ×%Ò%Ôin×ivÑivð 	ô 	‰Ñ˜°%ÐÒSr'   Úhidden_statesÚ
layer_pastÚattention_maskÚ	head_maskÚencoder_hidden_statesÚencoder_attention_maskÚ	use_cacheÚoutput_attentionsÚreturn.c	                 ó(  • U R                   " S
UUUUUUUS.U	D6n
 U
S   nU
SS  n[        U R                     R                  nUR                  S   nUR                  S   nUR                  S   n[        U R                   U5      " U R                  5      R                  USS9u  nnnUR                  SU R                  U R                  U R                   R                  5      R                  USSS5      R                  SS5      nUR                  SU R                  U R                  U R                   R                  5      R                  USSS5      R                  SS5      n[        U R                     R                  nU" U R                   XS9nUR                  n[         R"                  " UUR                  SS5      R%                  U5      5      [&        R(                  " U R                   R                  5      -  nU R*                  [,        R.                  " US[         R0                  S	9R%                  U5      -  n[         R"                  " UU5      R                  SS5      R3                  XïS5      nUU-   nUR%                  U5      nU4U-   nU$ )N)r?   rA   rB   rC   rD   rE   rF   r   r   é   ©Údim)r?   rC   é   éÿÿÿÿ©rK   r   © )r   r   r
   Úk_proj_layerÚshapeÚgetattrr   ÚsplitÚviewr   r   Úhead_dimÚrepeatÚ	transposeÚcompute_query_statesr   r   ÚmatmulÚtoÚmathÚsqrtr!   ÚFÚsoftmaxr-   Úreshape)r"   r?   r@   rA   rB   rC   rD   rE   rF   ÚkwargsÚattn_outputsÚattn_outputÚadd_outputsÚc_attn_layerÚbszÚq_lenÚ	embed_dimÚ_ÚkeyÚvalueÚ	adapter_kÚ	adapter_vrX   Úquery_statesÚprevious_dtypeÚscoresÚadapter_outputÚhidden_stateÚoutputs                                r%   ÚforwardÚAdaptedAttentionGPT.forwardG   su  € ð —z’zð 	
Ø'Ø)ØØ"7Ø#9ØØ/ñ	
ð ñ	
ˆð	ð # 1‘oˆØ" 1 2Ð&ˆä0°·±ÑA×NÑNˆà×Ñ Ñ"ˆØ×!Ñ! !Ñ$ˆØ×%Ñ% aÑ(ˆ	ä §
¡
¨LÔ9¸$×:NÑ:NÓO×UÑUÐV_ÐefÐUÐg‰ˆˆ3�ð �H‰H�Q˜×(Ñ(¨$¯.©.¸$¿*¹*×:MÑ:MÓN×UÑUÐVYÐ[\Ð^_ÐabÓc×mÑmÐnoÐqrÓsð 	ð �J‰J�q˜$×*Ñ*¨D¯N©N¸D¿J¹J×<OÑ<OÓP×WÑWÐX[Ð]^Ð`aÐcdÓe×oÑoÐpqÐstÓuð 	ô  9¸¿¹ÑI×^Ñ^ÐÙ+Ø�J‰J mñ
ˆð &×+Ñ+ˆä—’˜l¨I×,?Ñ,?ÀÀ1Ó,E×,HÑ,HÈÓ,XÓYÔ\`×\eÒ\eØ�J‰J×Ñó]
ñ 
ˆð
 ×#Ñ#¤a§i¢i°¸BÄeÇmÁmÑ&T×&WÑ&WÐXfÓ&gÑgˆäŸš f¨iÓ8×BÑBÀ1ÀaÓH×PÑPÐQTÐ]_Ó`ˆð # ^Ñ3ˆð $—‘ ~Ó6ˆð � ;Ñ.ˆØˆr'   rO   )NNNNNFF)r(   r)   r*   r+   r,   r   r   Útupler   ÚFloatTensorÚTensorÚboolr   rs   r0   r1   r2   s   @r%   r4   r4   >   s  ø† ÙOõTð 59Ø6:Ø15Ø8<Ø>BØ$)Ø,1ñDà  e×&7Ñ&7Ñ 8Ñ9ðDð ˜U 5§<¡<Ñ0Ñ1ðDð ! ×!2Ñ!2Ñ3ð	Dð
 ˜E×-Ñ-Ñ.ðDð  (¨¯©Ñ5ðDð !)¨×):Ñ):Ñ ;ðDð ˜D‘>ðDð $ D™>ðDð 
ˆu�U—\‘\ 5¨¯©Ñ#6Ð6Ñ7¸Ð<Ñ	=÷Dó Dr'   r4   c                   ó2   ^ • \ rS rSrSrU 4S jrS rSrU =r$ )ÚAdaptedAttentionéŽ   zGThis module wraps a LLamaAttention module and injects adaption prompts.c                 ó  >• UR                   R                  R                  [        R                  [        R
                  4;  a   UR                   R                  R                  O[        R                  n[        TU ]!  XX4S9  g r7   )	Úq_projr9   r   r   r:   r;   r-   r   r   r<   s        €r%   r   ÚAdaptedAttention.__init__‘   r>   r'   c                 ó”  • UR                  SS5      (       a  [        S5      eU R                  " S0 UD6tp#UR                  S   nUR                  S   nUR                  S   n[        U R
                     R                  n[        U R
                     R                  n[        U R
                     R                  n	U R                  R                  R                  U R                  R                  R                  -  n
Xx:X  a8  [        U R                  U5      " U R                  5      R                  USS9u  p;nOL[        U R                  U5      " U R                  5      n[        U R                  U5      " U R                  5      nU R                  R                  R                   nUR#                  SU R$                  XÚ-  U R                  R&                  5      R)                  USSS5      R+                  SS5      nUR#                  SU R$                  XÚ-  U R                  R&                  5      R)                  USSS5      R+                  SS5      n[,        R.                  " XêSS9n[,        R.                  " XúSS9n[        U R
                     R0                  nU" SS	U R                  0UD6nUR2                  n[,        R4                  " UUR+                  SS
5      R7                  U5      5      [8        R:                  " U R                  R&                  5      -  nU R<                  [>        R@                  " US[,        RB                  S9R7                  U5      -  n[,        R4                  " UU5      R+                  SS5      RE                  XES5      nU	b  [        U R                  U	5      " U5      nUU-   nUR7                  U5      nU/UQ7$ )a  
Forward pass for the adapter which wraps the original LlamaAttention module.

"Official" paper implementation:
https://github.com/ZrrSkywalker/LLaMA-Adapter/blob/41c3546fe1997ab8a65809dc8d8f9252b19d9faf/llama/model.py#L141

Args:
    kwargs: See the original LlamaAttention module.
Úoutput_attentionFz,output_attention is not currently supported.r   r   rI   rJ   )ÚrepeatsrK   r   rL   rM   rN   rO   )#ÚgetÚNotImplementedErrorr   rQ   r   r
   rP   Úv_proj_layerÚo_proj_layerÚk_projÚin_featuresÚout_featuresrR   r   rS   r   r   rT   r   rU   rV   rW   r   Úrepeat_interleaverX   r   rY   rZ   r[   r\   r!   r]   r^   r-   r_   )r"   r`   rr   rh   re   rf   rg   rP   r„   r…   Úfactorri   rj   r   rk   rl   rX   rm   rn   ro   rp   s                        r%   rs   ÚAdaptedAttention.forward—   s  € ð �:‰:Ð(¨%×0Ñ0Ü%Ð&TÓUÐUà—Z’ZÑ) &Ñ)ˆ
ˆØ�l‰l˜1‰oˆØ—‘˜Q‘ˆØ—L‘L ‘Oˆ	Ü0°·±ÑA×NÑNˆÜ0°·±ÑA×NÑNˆÜ0°·±ÑA×NÑNˆà�J‰J×Ñ×)Ñ)¨T¯Z©Z×->Ñ->×-KÑ-KÑKð 	ð Ó'Ü# D§J¡J°Ô=¸d×>RÑ>RÓS×YÑYÐZcÐijÐYÐk‰MˆA‘Eä˜$Ÿ*™* lÔ3°D×4HÑ4HÓIˆCÜ˜DŸJ™J¨Ô5°d×6JÑ6JÓKˆEà—J‘J×%Ñ%×9Ñ9ˆ	ð �H‰H�Q˜×(Ñ(¨9Ñ+>ÀÇÁ×ATÑATÓUß‰V�C˜˜A˜qÓ!ß‰Y�q˜!‹_ð 	ð �J‰J�q˜$×*Ñ*¨YÑ-@À4Ç:Á:×CVÑCVÓWß‰V�C˜˜A˜qÓ!ß‰Y�q˜!‹_ð 	ô ×+Ò+¨IÈ1ÑMˆ	Ü×+Ò+¨IÈ1ÑMˆ	ä8¸¿¹ÑI×^Ñ^Ðá+ÑG°$·*±*ÐGÀÑGˆà%×+Ñ+ˆô —’˜l¨I×,?Ñ,?ÀÀ1Ó,E×,HÑ,HÈÓ,XÓYÔ\`×\eÒ\eØ�J‰J×Ñó]
ñ 
ˆð
 ×#Ñ#¤a§i¢i°¸BÄeÇmÁmÑ&T×&WÑ&WÐXfÓ&gÑgˆäŸš f¨iÓ8×BÑBÀ1ÀaÓH×PÑPÐQTÐ]_Ó`ˆð Ñ#Ü$ T§Z¡Z°Ô>¸~ÓNˆNð ˜.Ñ(ˆð —‘˜>Ó*ˆØˆz˜‰zÐr'   rO   )	r(   r)   r*   r+   r,   r   rs   r0   r1   r2   s   @r%   rz   rz   Ž   s   ø† ÙQõT÷Hð Hr'   rz   )r[   Útypingr   r   r   Útorch.nnr   Útorch.nn.functionalÚ
functionalr]   r   r   ÚModuler   r4   rz   rO   r'   r%   Ú<module>r‘      sN   ðó ß "ã Ý ß Ð å -ô"]˜BŸI™Iô "]ôJMÐ/ô Mô`QÐ,õ Qr'   