ó
    pyüió  ã                   óê  • S SK r SSKJrJrJr  SSKJr  \R                  " \5      r	\" SSS9r
\" SSS9r\" 5       r\" 5       rS	\ R                  S
\S\ R                  4S jrS\ R                  S-  S\ R                  S\4S jr   SS\ R&                  R(                  S\ R                  S\ R                  S\ R                  S\ R                  S-  S\S\S-  S\S-  S\\ R                  S4   4S jjrg)é    Né   )Úis_torch_npu_availableÚis_torch_xpu_availableÚlogging)Úis_torch_greater_or_equalz2.5T)Ú
accept_devz2.8Úhidden_statesÚn_repÚreturnc                 ó    • U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
é   N)ÚshapeÚexpandÚreshape)r	   r
   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         Úe/home/mande/repo/quber/.venv/lib/python3.13/site-packages/transformers/integrations/sdpa_attention.pyÚ	repeat_kvr      s_   € ð
 2?×1DÑ1DÑ.€E Ø�ƒzØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÑ(CÀTÓTÐTó    Úattention_maskÚkeyc                 óF   • [         (       a  [        $ [        =(       a    U S L $ )N)Ú_is_torch_xpu_availableÚ#_is_torch_greater_or_equal_than_2_8Ú#_is_torch_greater_or_equal_than_2_5)r   r   s     r   Úuse_gqa_in_sdpar      s    € ÷ ÒÜ2Ð2Ü.×I°>ÀTÐ3IÐIr   ÚmoduleÚqueryÚvalueÚdropoutÚscalingÚ	is_causalc                 ó�  • UR                  SS5      (       a  [        R                  S5        0 n	[        U S5      (       a?  [	        XB5      (       d+  [        X R                  5      n[        X0R                  5      nOSS0n	Ub  UO[        U SS5      nUR                  S   S	:„  =(       a    US L =(       a    Un[        R                  R                  5       (       a/  [        U[        R                  5      (       a  UR                  5       n[        (       a^  Ub[  UR                   [        R"                  :w  a=  [        R$                  " UR#                  5       5      R'                  UR(                  5      n[        R*                  R,                  R.                  " UUU4UUUUS
.U	D6n
U
R1                  S	S5      R3                  5       n
U
S 4$ )NÚoutput_attentionsFzƒ`sdpa` attention does not support `output_attentions=True`. Please set your attention to `eager` if you want any of these features.Únum_key_value_groupsÚ
enable_gqaTr$   r   r   )Ú	attn_maskÚ	dropout_pÚscaler$   )ÚgetÚloggerÚwarning_onceÚhasattrr   r   r'   Úgetattrr   ÚtorchÚjitÚ
is_tracingÚ
isinstanceÚTensorÚitemÚ_is_torch_npu_availableÚdtypeÚboolÚlogical_notÚtoÚdeviceÚnnÚ
functionalÚscaled_dot_product_attentionÚ	transposeÚ
contiguous)r   r    r   r!   r   r"   r#   r$   ÚkwargsÚsdpa_kwargsÚattn_outputs              r   Úsdpa_attention_forwardrE   (   s�  € ð ‡z�zÐ% u×-Ñ-Ü×ÑðWô	
ð €KÜˆvÐ-×.Ñ.Ü˜~×3Ñ3Ü˜C×!<Ñ!<Ó=ˆCÜ˜e×%@Ñ%@ÓA‰Eà'¨Ð.ˆKð 'Ñ2‘	¼ÀÈÐUYÓ8Z€Ið —‘˜A‘ Ñ"×K ~¸Ð'=×KÀ)€Iô ‡y�y×Ñ×Ñ¤*¨Y¼¿¹×"EÑ"EØ—N‘NÓ$ˆ	÷
 ÒØÑ%¨.×*>Ñ*>Ä%Ç*Á*Ó*Lä"×.Ò.¨~×/BÑ/BÓ/DÓE×HÑHÈÏÉÓVˆNä—(‘(×%Ñ%×BÒBØØØð	ð !ØØØñ	ð ñ	€Kð ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜ÐÐr   )g        NN)r1   Úutilsr   r   r   Úutils.import_utilsr   Ú
get_loggerÚ__name__r-   r   r   r   r7   r5   Úintr   r9   r   r=   ÚModuleÚfloatÚtuplerE   © r   r   Ú<module>rO      sH  ðÛ ç KÑ KÝ :ð 
×	Ò	˜HÓ	%€ñ '@ÀÐRVÑ&WÐ #Ù&?ÀÐRVÑ&WÐ #Ù0Ó2Ð Ù0Ó2Ð ð	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ô 	Uð	J E§L¡L°4Ñ$7ð 	J¸e¿l¹lð 	JÈtô 	Jð$ Ø Ø!ñ@Ø�H‰H�O‰Oð@à�<‰<ð@ð 
�‰ð@ð �<‰<ð	@ð
 —L‘L 4Ñ'ð@ð ð@ð �T‰\ð@ð �d‰{ð@ð ˆ5�<‰<˜ÐÑö@r   