ó
    !EñiK  ã                  ó„  • % S r SSKJr  SSKrSSKrSSKJr  \(       a  SSKJr  SSK	J
r
  SSKJr  SSKJrJr  SSKrSS	KJr  S
SKJr  S/rSqS\S'   SqS\S'   \
 " S S5      5       r\S*S j5       r S+   S,S jjrS-S jrS.S jr                S/S jr                        S0S jr                      S1S jr \" S5      r!S2S jr"S3S jr#    S4                                 S5S jjr$ S6                               S7S jjr%   S8SSSSSSS.                                     S9S  jjjr&SSSSSSS.                               S:S! jjr'SSSS".                                 S;S# jjr(      S<SS$.                   S=S% jjjr)   S>SS$.             S?S& jjjr*SS$.                             S@S' jjr+\RX                  " S(\S)9  g)Az�
PROTOTYPE!
Flash Attention 3 implementation.
For fp8: only supports forward pass right now.
For fp16/bf16: supports forward and backward pass.
é    )ÚannotationsN)ÚTYPE_CHECKING)ÚCallable)Ú	dataclass)Úcache)ÚTypeVarTupleÚUnpack)ÚLibraryé   )Ú	_registryÚregister_flash_attention_fa3zCallable | NoneÚ_FA3_CUDA_FWDÚ_FA3_CUDA_BWDc                  ó*   • \ rS rSr% S\S'   SS jrSrg)Ú
_FA3Handleé&   zLibrary | NoneÚlibraryc                óP   • S U l         [        R                  R                  S5        g )NF)r   ÚtorchÚ_CÚ_set_sdp_use_fa3)Úselfs    ÚT/home/mande/repo/quber/.venv/lib/python3.13/site-packages/torch/nn/attention/_fa3.pyÚremoveÚ_FA3Handle.remove*   s   € ØˆŒä�‰×!Ñ! %Õ(ó    )r   N)ÚreturnÚNone)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__annotations__r   Ú__static_attributes__© r   r   r   r   &   s   ‡ àÓ÷)r   r   c                óH   • [         R                  R                  U 5      u  pU$ ©N)r   ÚcudaÚget_device_capability)ÚdeviceÚmajorÚ_s      r   Ú_get_device_majorr-   0   s   € ä�z‰z×/Ñ/°Ó7�H€EØ€Lr   c                ó|   • [        U 5        [        R                  R                  S5        [	        [        5       5      $ )z‰
Register FA3 flash attention kernels with the PyTorch dispatcher.

Args:
    module_path: Python module path to the FA3 implementation.
T)Ú_fa3_import_moduler   r   r   r   Ú_fa3_register_kernels©Úmodule_paths    r   r   r   6   s/   € ô �{Ô#ô 
‡H�H×Ñ˜dÔ#äÔ+Ó-Ó.Ð.r   c                óü  • [         R                  " U 5        [        [        R                  S5      (       d  [        SU  S35      e[        [        R                  R                  S5      (       d  [        SU  S35      e[        [        R                  R                  S5      (       d  [        SU  S35      e[        R                  R                  R                  q[        R                  R                  R                  q
g )NÚflash_attn_3zModule 'z' does not expose FA3 kernelsÚfwdz%' does not expose FA3 forward kernelsÚbwdz&' does not expose FA3 backward kernels)Ú	importlibÚimport_moduleÚhasattrr   ÚopsÚRuntimeErrorr4   r5   r   r6   r   r1   s    r   r/   r/   G   sÆ   € Ü×Ò˜KÔ(Ü”5—9‘9˜n×-Ñ-Ü˜X k ]Ð2OÐPÓQÐQÜ”5—9‘9×)Ñ)¨5×1Ñ1ÜØ�{�mÐ#HÐIó
ð 	
ô ”5—9‘9×)Ñ)¨5×1Ñ1ÜØ�{�mÐ#IÐJó
ð 	
ô —I‘I×*Ñ*×.Ñ.€MÜ—I‘I×*Ñ*×.Ñ.�Mr   c                 ó4  • [        SSS5      n U R                  S[        S5        U R                  S[        S5        U R                  S[        S5        U R                  S[
        S5        U R                  S[        S5        U R                  S	[        S5        U $ )
NÚatenÚIMPLÚCUDAz"_flash_attention_forward.quantizedz-_scaled_dot_product_flash_attention.quantizedÚ_flash_attention_forwardÚ#_scaled_dot_product_flash_attentionÚ_flash_attention_backwardÚ,_scaled_dot_product_flash_attention_backward)r
   ÚimplÚ!_fa3_flash_attention_forward_implÚ4_fa3_scaled_dot_product_flash_attention_forward_implÚ)_fa3_flash_attention_forward_impl_defaultÚ<_fa3_scaled_dot_product_flash_attention_forward_impl_defaultÚ"_fa3_flash_attention_backward_implÚ5_fa3_scaled_dot_product_flash_attention_backward_impl)Úlibs    r   r0   r0   X   s™   € Ü
�&˜& &Ó
)€CØ‡H�HØ,Ô.OÐQWôð ‡H�HØ7Ü<Øôð
 ‡H�HØ"Ô$MÈvôð ‡H�HØ-ÜDØôð ‡H�HÐ(Ô*LÈfÔUØ‡H�HØ6Ü=Øôð
 €Jr   c                ó   • US:w  a  g[        S U 5       5      (       d  g[        U Vs1 s H  owR                  iM     sn5      S:w  a  gU R                  [        R
                  :X  a$  Ub  Ub  Uc  [        R                  " S[        5        Uc  U R                  5       S:w  a  g	Ub  U R                  5       S
:w  a  g[        R                  R                  5       (       d  g[        U R                  5      S:w  a  gg s  snf )Nç        zdropout_p must be 0c              3  ó8   #   • U  H  oR                   v •  M     g 7fr'   )Úis_cuda©Ú.0Úts     r   Ú	<genexpr>Ú,_fa3_common_support_error.<locals>.<genexpr>€   s   é € Ð*¢'˜Q�yŽy¢'ùs   ‚zinputs must be CUDA tensorsr   úinputs must share devicez¯When using SDPA with fp8, descale tensor should always be used for accurate dequantization. Please use _scaled_dot_product_attention_quantized and provide the descale tensors.é   zdense query must be 4Dé   zragged query must be 3DzCUDA not availableé	   z#FA3 requires compute capability 9.0)ÚallÚlenr*   Údtyper   Úfloat8_e4m3fnÚwarningsÚwarnÚUserWarningÚdimr(   Úis_availabler-   )ÚqueryÚtensorsÚ	dropout_pÚ	cum_seq_qÚ	q_descaleÚ	k_descaleÚ	v_descalerR   s           r   Ú_fa3_common_support_errorri   t   sÜ   € ð �CÓØ$äÑ*¡'Ó*×*Ñ*Ø,Ü
™gÓ&šg˜�HŒH™gÑ&Ó'¨1Ó,Ø)Ø‡{�{”e×)Ñ)Ó)ØÑ˜YÑ.°)Ñ2Cä�Šð+ô ô	
ð Ñ˜UŸY™Y›[¨AÓ-Ø'ØÑ §¡£°Ó!1Ø(Ü�:‰:×"Ñ"×$Ñ$Ø#Ü˜Ÿ™Ó&¨!Ó+Ø4Øùò) 's   ©C;c           	     óÆ  ^• U(       a  gUb  gUb1  UR                   [        R                  :w  a  gUR                  (       d  g[        R                  [        R
                  [        R                  4m[        U4S jXU1 5       5      (       d  ST 3$ [        XU1 Vs1 s H  o»R                   iM     sn5      S:w  a  g[        U XU4UUUU	U
5      nUb	  US	:X  a  g
U$ g s  snf )Nzreturn_debug_mask must be Falsezalibi_slopes not supportedzseqused_k must be int32zseqused_k must be CUDAc              3  ó@   >#   • U  H  oR                   T;   v •  M     g 7fr'   ©r[   ©rQ   rR   Úsupported_dtypess     €r   rS   Ú-_fa3_forward_support_error.<locals>.<genexpr>°   s   øé € ÐHÒ4G¨q�w‰wÐ*Ö*Ò4Gùó   ƒúinputs must be one of r   ú#all inputs must have the same dtyperU   z(query, key, value must be on same device)
r[   r   Úint32rO   r\   Úfloat16Úbfloat16rY   rZ   ri   )rb   ÚkeyÚvaluerd   Úreturn_debug_maskÚalibi_slopesÚ	seqused_kre   rf   rg   rh   rR   Úerrorrn   s                @r   Ú_fa3_forward_support_errorr|   ™   sæ   ø€ ö Ø0ØÑØ+ØÑØ�?‰?œeŸk™kÓ)Ø,Ø× × Ø+Ü×+Ñ+¬U¯]©]¼E¿N¹NÐKÐÜÔH°UÀÑ4GÓH×HÑHØ'Ð(8Ð'9Ð:Ð:Ü
˜e¨%Ñ0Ó1Ò0˜�GŒGÑ0Ñ1Ó2°aÓ7Ø4Ü%ØØ	�UÐØØØØØó€Eð ÑØÐ.Ó.Ø=ØˆØùò 2s   ÂCc
           	     óž  ^• UR                   [        R                  :X  a   gUR                   [        R                  :w  a  g[        R                  [        R
                  4m[        U4S jXX#U1 5       5      (       d  ST 3$ [        XX#U1 V
s1 s H  oªR                   iM     sn
5      S:w  a  g[        UXX#XE4UUS S S 5      nUb  U$ g s  sn
f )NzHFA3 backward does not support fp8 - use inference only (torch.no_grad())zlogsumexp dtype must be float32c              3  ó@   >#   • U  H  oR                   T;   v •  M     g 7fr'   rl   rm   s     €r   rS   Ú._fa3_backward_support_error.<locals>.<genexpr>Ø   s   øé € ÐWÒ4V¨q�w‰wÐ*Ö*Ò4Vùrp   rq   r   rr   )	r[   r   r\   Úfloat32rt   ru   rY   rZ   ri   )Úgrad_outrb   rv   rw   ÚoutÚ	logsumexprd   re   Úwindow_size_leftÚwindow_size_rightrR   r{   rn   s               @r   Ú_fa3_backward_support_errorr†   Ä   sÕ   ø€ ð ‡{�{”e×)Ñ)Ó)àVð	
ð ‡�œ%Ÿ-™-Ó'Ø0ÜŸ™¤u§~¡~Ð6ÐÜÔW°XÀcÐRUÑ4VÓW×WÑWØ'Ð(8Ð'9Ð:Ð:Ü
˜h¨s¸3Ñ?Ó@Ò?˜�GŒGÑ?Ñ@ÓAÀQÓFØ4Ü%ØØ	˜# cÐ5ØØØØØó€Eð ÑØˆØùò As   ÂC
ÚTsc                 ó&   • [        S U  5       5      $ )Nc              3  óD   #   • U  H  oR                  S S5      v •  M     g7f)r   é   N)Ú	transposerP   s     r   rS   Ú#_transpose_dense.<locals>.<genexpr>î   s   é € Ð4ªG q—‘˜Q ×"Ð"ªGùs   ‚ )Útuple)rc   s    r   Ú_transpose_denserŽ   í   s   € ÜÑ4©GÓ4Ó4Ð4r   c                óV   • U b%  U R                  S5      S:w  a  U R                  5       $ U $ )z2Ensure tensor is contiguous in the last dimension.éÿÿÿÿr   )ÚstrideÚ
contiguous)Úxs    r   Ú_maybe_contiguousr”   ñ   s&   € à™]¨q¯x©x¸«|¸qÓ/@ˆ1�<‰<‹>ÐGÀaÐGr   c                ó¸  • [         c  [        S5      e[        U 5      n[        U5      nUR                  [        R
                  :X  a:  UR                  S5      S:w  a%  UR                  S5      S:w  a  UR                  5       O
[        U5      n[        U5      n[        U5      n[        U5      n[        / UPUPUPSPSPSPUPUPUPSPSPUPUPUPSPSPSPSPSPSPUPUPUPUPUPU	b  U	OSPU
b  U
OSPSPSPSPSP[        R                  " 5       (       a  SOSPSP[        R                  R                  5       =(       d    SP76 u  nnnnUUR                  5       4$ )	z>
Run the FA3 forward pass by calling the C++ kernel directly.
NúFA3 not registeredr�   r   éýÿÿÿr   rM   T)r   r;   r”   r[   r   r\   r‘   r’   Ú$are_deterministic_algorithms_enabledr   Ú_get_sm_carveout_experimental)rb   rv   rw   Úcu_seq_qÚcu_seq_kÚmax_qÚmax_kÚscaleÚ	is_causalr„   r…   rz   r‚   rf   rg   rh   ÚqÚkÚvÚcu_seqlens_qÚcu_seqlens_kÚsoftmax_lseÚ	out_accumÚsoftmax_lse_accums                           r   Ú_fa3_run_forwardr¨   ö   sP  € ô* ÑÜÐ/Ó0Ð0ä˜%Ó €AÜ˜#Ó€Að �;‰;œ%×-Ñ-Ó-Ø�L‰L˜Ó Ó!Ø�L‰L˜Ó Ó!ð 	×ÑÔô ˜uÓ%ð ô % XÓ.€LÜ$ XÓ.€LÜ! )Ó,€Iä5Bð #6Ø	ð#6à	ð#6ð 	
ð#6ð 	ð	#6ð
 	ð#6ð 	ð#6ð 	ð#6ð 	ð#6ð 	ð#6ð 	ð#6ð 	ð#6ð 	ð#6ð 	ð#6ð 	ð#6ð 	ð#6ð  	ð!#6ð" 	ð##6ð$ 	ð%#6ð& 	ð'#6ð( 	ð)#6ð* 	ð+#6ð, 	ð-#6ð. 	ð/#6ð0 	ð1#6ð2 	ð3#6ð4 -Ñ8Ñ¸bð5#6ð6 /Ñ:ÑÀð7#6ð8 	
ð9#6ð: 	ð;#6ð< 	ð=#6ð> 	ð?#6ô@ ×7Ò7×9Ñ9‰¸qðA#6ðB 	ðC#6ôD 	�‰×.Ñ.Ó0×5°AòE#6Ñ2€Cˆ�iÐ!2ðH �×&Ñ&Ó(Ð(Ð(r   c                ój  • [         c  [        S5      e[        U 5      nUR                  S5      S:w  a  UR	                  5       OUnUR                  S5      S:w  a  UR	                  5       OUnUR                  S5      S:w  a  UR	                  5       OUn[        U5      n[        U5      n[
        R                  " U5      n[
        R                  " U5      n[
        R                  " U5      n[        UUUUUUUUUUUS S UU	U
UUUSU[
        R                  R                  5       =(       d    S5        UUU4$ )Nr–   r�   r   rM   r   )	r   r;   r”   r‘   r’   r   Ú
empty_liker   r™   )r�   rb   rv   rw   r‚   rƒ   rš   r›   Úmax_seqlen_qÚmax_seqlen_krž   rŸ   r„   r…   ÚdeterministicÚdoutr    r¡   r¢   ÚoÚlseÚdqÚdkÚdvs                           r   Ú_fa3_run_backwardr´   C  s/  € ô" ÑÜÐ/Ó0Ð0ô ˜XÓ&€DØ#Ÿl™l¨2Ó.°!Ó3ˆ×ÑÔ¸€AØŸJ™J r›N¨aÓ/ˆ�‰Ô°S€AØ#Ÿl™l¨2Ó.°!Ó3ˆ×ÑÔ¸€AÜ˜#Ó€AÜ
˜IÓ
&€Cô 
×	Ò	˜!Ó	€BÜ	×	Ò	˜!Ó	€BÜ	×	Ò	˜!Ó	€BÜØØ	Ø	Ø	Ø	ØØ
Ø
Ø
ØØØØØØØØØØØØÜ�‰×.Ñ.Ó0×5°Aô-ð0 ˆr�2ˆ:Ðr   r�   ©rž   r„   r…   rz   ry   r‚   c               ó¢  • [        U UUUU	UUUU
UU5      nUb  [        SU 35      e[        U UUUUUUUUUUUUU
UU5      u  nn[        R                  " S[        R
                  U R                  S9n[        R                  " S[        R
                  U R                  S9n[        R                  " SU R                  U R                  S9nUUUUU4$ )Nz)FA3 flash_attention forward unsupported: )rŠ   )r[   r*   r%   r   )	r|   r;   r¨   r   ÚzerosÚuint64r*   Úemptyr[   )rb   rv   rw   re   Ú	cum_seq_krœ   r�   rd   rŸ   rx   rf   rg   rh   rž   r„   r…   rz   ry   r‚   r{   r°   Ú	rng_stateÚphilox_offsetÚ
debug_masks                           r   rE   rE   ~  sé   € ô, 'ØØØØØØØØØØØó€Eð ÑÜÐFÀuÀgÐNÓOÐOÜØØØØØØØØØØØØØØØØó!�H€Cˆô$ —’˜D¬¯©¸U¿\¹\ÑJ€IÜ—K’K ¬%¯,©,¸u¿|¹|ÑL€MÜ—’˜Q e§k¡k¸%¿,¹,ÑG€JØ��Y ¨zÐ9Ð9r   c
               ó8   • [        U UUUUUUUUU	S S S U
UUUUUS9$ )Nrµ   )rE   )rb   rv   rw   re   rº   rœ   r�   rd   rŸ   rx   rž   r„   r…   rz   ry   r‚   s                   r   rG   rG   »  sJ   € ô& -ØØØØØØØØØØØØØØØ)Ø+ØØ!Øñ'ð r   )rž   r„   r…   c               óÎ   • [        U UUUUUU
UUU5
      nUb  [        SU 35      e[        R                  " 5       n[	        U UUUUUUUUU	UUUb  UOSUb  UOSU5      u  nnnUUU4$ )z0FA3 implementation of _flash_attention_backward.z*FA3 flash_attention backward unsupported: r�   )r†   r;   r   r˜   r´   )r�   rb   rv   rw   r‚   rƒ   re   rº   rœ   r�   rd   rŸ   r»   Úunusedrž   r„   r…   r{   r­   r±   r²   r³   s                         r   rI   rI   å  s°   € ô* (ØØØØØØØØØØó€Eð ÑÜÐGÈÀwÐOÓPÐPä×>Ò>Ó@€Mä"ØØØØØØØØØØØØØ,Ñ8Ñ¸bØ.Ñ:ÑÀØó�J€BˆˆBð" ˆr�2ˆ:Ðr   ©rž   c	               ó  • [        U UUUUS S S UUU5      n
U
b  [        SU
 35      e[        XU5      u  p¼nU R                  [        R
                  :X  a  [        R                  OU R                  n[        R                  " XS9nUR                  SS5      nUR                  S5      nUR                  S5      n[        UUUS S UUUUUU	UUUUS9u  nnnnnU R                  S5      nUR                  S5      nUUS S UUUUU4	$ )NzFA3 SDPA forward unsupported: rl   r   rŠ   )rž   r‚   rf   rg   rh   )r|   r;   rŽ   r[   r   r\   ru   rª   r‹   ÚsizerE   )rb   rv   rw   rf   rg   rh   rd   rŸ   rx   rž   r{   r    r¡   r¢   Ú	out_dtypeÚout_bhsdÚout_bshdÚmax_q_flashÚmax_k_flashr,   r°   r»   r¼   r½   rœ   r�   s                             r   rF   rF      s;  € ô 'ØØØØØØØØØØØó€Eð ÑÜÐ;¸E¸7ÐCÓDÐDÜ˜u¨5Ó1�G€Aˆ!ð
 #(§+¡+´×1DÑ1DÓ"D”—’È%Ï+É+€IÜ×Ò Ñ7€HØ×!Ñ! ! QÓ'€Hà—&‘&˜“)€KØ—&‘&˜“)€KÜ3TØ	Ø	Ø	ØØØØØØØØØØØØñ4Ñ0€A€sˆI�} jð" �J‰J�q‹M€EØ�H‰H�Q‹K€EàØØØØØØØØð
ð 
r   c               ó&   • [        U UUS S S UUUUS9
$ )NrÁ   )rF   )rb   rv   rw   rd   rŸ   rx   rž   s          r   rH   rH   g  s0   € ô @ØØØØØØØØØØñð r   c               óÎ   • [        XX#XEU
SSS5
      nUb  [        SU 35      e[        XX#U5      u  nnnnn[        UUUUUUSSUU	U
UUUUS9u  nnn[        UUU5      u  nnnUUU4$ )zCFA3 implementation of _scaled_dot_product_flash_attention_backward.NzFA3 SDPA backward unsupported: rÁ   )r†   r;   rŽ   rI   )r�   rb   rv   rw   r‚   rƒ   re   rº   rœ   r�   rd   rŸ   Úphilox_seedr¼   rž   r{   Ú
grad_out_tÚq_tÚk_tÚv_tÚout_tr±   r²   r³   Údq_outÚdk_outÚdv_outs                              r   rJ   rJ     s¹   € ô& (Ø˜ S°YÀÀdÈDó€Eð ÑÜÐ<¸U¸GÐDÓEÐEô (8Ø˜ Só(Ñ$€J��S˜#˜uô 4ØØØØØØØØØØØØØØØñ�J€BˆˆBô& .¨b°"°bÓ9Ñ€FˆF�FØ�6˜6Ð!Ð!r   ÚFA3)Úregister_fn)r*   ztorch.devicer   Úint)Úflash_attn_interface)r2   Ústrr   r   )r2   rØ   r   r   )r   r
   )rb   útorch.Tensorrc   ztuple[torch.Tensor, ...]rd   Úfloatre   útorch.Tensor | Nonerf   rÛ   rg   rÛ   rh   rÛ   r   ú
str | None)rb   rÙ   rv   rÙ   rw   rÙ   rd   rÚ   rx   Úboolry   rÛ   rz   rÛ   re   rÛ   rf   rÛ   rg   rÛ   rh   rÛ   r   rÜ   )r�   rÙ   rb   rÙ   rv   rÙ   rw   rÙ   r‚   rÙ   rƒ   rÙ   rd   rÚ   re   rÛ   r„   ú
int | Noner…   rÞ   r   rÜ   )rc   z
Unpack[Ts]r   ztuple[Unpack[Ts]])r“   rÛ   r   rÛ   )NNNN)"rb   rÙ   rv   rÙ   rw   rÙ   rš   rÛ   r›   rÛ   rœ   rÖ   r�   rÖ   rž   úfloat | NonerŸ   rÝ   r„   rÞ   r…   rÞ   rz   rÛ   r‚   rÛ   rf   rÛ   rg   rÛ   rh   rÛ   r   z!tuple[torch.Tensor, torch.Tensor])F) r�   rÙ   rb   rÙ   rv   rÙ   rw   rÙ   r‚   rÙ   rƒ   rÙ   rš   rÛ   r›   rÛ   r«   rÞ   r¬   rÞ   rž   rß   rŸ   rÝ   r„   rÖ   r…   rÖ   r­   rÝ   r   z/tuple[torch.Tensor, torch.Tensor, torch.Tensor])NNN)&rb   rÙ   rv   rÙ   rw   rÙ   re   rÛ   rº   rÛ   rœ   rÖ   r�   rÖ   rd   rÚ   rŸ   rÝ   rx   rÝ   rf   rÛ   rg   rÛ   rh   rÛ   rž   rß   r„   rÖ   r…   rÖ   rz   rÛ   ry   rÛ   r‚   rÛ   ) rb   rÙ   rv   rÙ   rw   rÙ   re   rÛ   rº   rÛ   rœ   rÖ   r�   rÖ   rd   rÚ   rŸ   rÝ   rx   rÝ   rž   rß   r„   rÖ   r…   rÖ   rz   rÛ   ry   rÛ   r‚   rÛ   )"r�   rÙ   rb   rÙ   rv   rÙ   rw   rÙ   r‚   rÙ   rƒ   rÙ   re   rÛ   rº   rÛ   rœ   rÖ   r�   rÖ   rd   rÚ   rŸ   rÝ   r»   rÙ   rÀ   rÙ   rž   rß   r„   rÞ   r…   rÞ   )NNNrM   FF)rb   rÙ   rv   rÙ   rw   rÙ   rf   rÛ   rg   rÛ   rh   rÛ   rd   rÚ   rŸ   rÝ   rx   rÝ   rž   rß   )rM   FF)rb   rÙ   rv   rÙ   rw   rÙ   rd   rÚ   rŸ   rÝ   rx   rÝ   rž   rß   )r�   rÙ   rb   rÙ   rv   rÙ   rw   rÙ   r‚   rÙ   rƒ   rÙ   re   rÛ   rº   rÛ   rœ   rÖ   r�   rÖ   rd   rÚ   rŸ   rÝ   rË   rÙ   r¼   rÙ   rž   rß   )-Ú__doc__Ú
__future__r   r7   r]   Útypingr   Úcollections.abcr   Údataclassesr   Ú	functoolsr   Útyping_extensionsr   r	   r   Útorch.libraryr
   Ú r   Ú__all__r   r#   r   r   r-   r   r/   r0   ri   r|   r†   r‡   rŽ   r”   r¨   r´   rE   rG   rI   rF   rH   rJ   Úregister_flash_attention_implr%   r   r   Ú<module>rë      sY  ðòõ #ã Û Ý  ö Ý(å !Ý ß 2ã Ý !å ð #ð€ð
 "&€ˆÓ %Ø!%€ˆÓ %ð ÷)ð )ó ð)ð óó ðð .ð/Øð/àõ/ô"/ô"ð8"Øð"à%ð"ð ð"ð #ð	"ð
 #ð"ð #ð"ð #ð"ð ô"ðJ(Øð(à	ð(ð ð(ð ð	(ð
 ð(ð &ð(ð #ð(ð #ð(ð #ð(ð #ð(ð #ð(ð ô(ðV#Øð#àð#ð 
ð#ð ð	#ð
 
ð#ð ð#ð ð#ð #ð#ð !ð#ð "ð#ð ô#ñL �$Ó€ô5ôHð$  $Ø%)Ø%)Ø%)ð!J)ØðJ)à	ðJ)ð ðJ)ð "ð	J)ð
 "ðJ)ð ðJ)ð ðJ)ð ðJ)ð ðJ)ð !ðJ)ð "ðJ)ð #ðJ)ð 
ðJ)ð #ðJ)ð #ðJ)ð  #ð!J)ð" 'õ#J)ðx  ð8Øð8àð8ð 
ð8ð ð	8ð
 
ð8ð ð8ð "ð8ð "ð8ð ð8ð ð8ð ð8ð ð8ð ð8ð ð8ð ð8ð  5õ!8ðL &*Ø%)Ø%)ð::ð ØØØ%)Ø(,Ø#ñ)::Øð::à	ð::ð ð::ð #ð	::ð
 #ð::ð ð::ð ð::ð ð::ð ð::ð ð::ð #ð::ð #ð::ð #ð::ð ð::ð  ð!::ð" ð#::ð$ #ð%::ð& &ð'::ð( 
ö)::ðR ØØØ%)Ø(,Ø#ñ#'Øð'à	ð'ð ð'ð #ð	'ð
 #ð'ð ð'ð ð'ð ð'ð ð'ð ð'ð ð'ð ð'ð ð'ð #ð'ð  &ð!'ð" 
õ#'ðt Ø#'Ø$(ñ%8Øð8àð8ð 
ð8ð ð	8ð
 
ð8ð ð8ð #ð8ð #ð8ð ð8ð ð8ð ð8ð ð8ð ð8ð ð8ð  ð!8ð" !ð#8ð$ "õ%8ð~ &*Ø%)Ø%)ØØØ#ðDð ñDØðDà	ðDð ðDð #ð	Dð
 #ðDð #ðDð ðDð ðDð ðDð öDðV ØØ#ðð ñØðà	ðð ðð ð	ð
 ðð ðð öðP ñ!2"Øð2"àð2"ð 
ð2"ð ð	2"ð
 
ð2"ð ð2"ð #ð2"ð #ð2"ð ð2"ð ð2"ð ð2"ð ð2"ð ð2"ð  ð2"ð  õ!2"ðj 
× 'Ò '¨Ð;WÓ Xr   