ó
    !Eñi•/  ã                  ó|  • % S r SSKJr  SSKrSSKJr  SSKJr  SSKJ	r	J
r
  SSKJrJr  SS	KJr  \
(       a  SS
KJr  SSKrSSKJr  S/rSqS\S'   \ " S S5      5       r\S%S j5       r S&   S'S jjr\S(S j5       rS)S jr S*         S+S jjr                  S,S jr                      S-S jr\" S5      r S.S jr! S/                       S0S jjr" S1                       S2S jjr#SSSSSSS.                               S3S jjr$SSSS.                                 S4S jjr%   S5SS .             S6S! jjjr&SS .                             S7S" jjr'\RP                  " S#\S$9  g)8zUBER PROTOTYPE!!!é    )ÚannotationsN)Ú	dataclass)Úcache)ÚAnyÚTYPE_CHECKING)ÚTypeVarTupleÚUnpacké   )Ú	_registry)Ú
ModuleType)ÚLibraryÚregister_flash_attention_fa4ú
str | NoneÚ_FA4_MODULE_PATHc                  ó*   • \ rS rSr% S\S'   SS jrSrg)Ú
_FA4Handleé   zLibrary | NoneÚlibraryc                ó   • S U l         g ©N©r   )Úselfs    ÚT/home/mande/repo/quber/.venv/lib/python3.13/site-packages/torch/nn/attention/_fa4.pyÚremoveÚ_FA4Handle.remove"   s	   € Øˆ�ó    r   N)ÚreturnÚNone)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__annotations__r   Ú__static_attributes__© r   r   r   r      s   ‡ àÓ÷r   r   c                óH   • [         R                  R                  U 5      u  pU$ r   )ÚtorchÚcudaÚget_device_capability)ÚdeviceÚmajorÚ_s      r   Ú_get_device_majorr-   &   s   € ä�z‰z×/Ñ/°Ó7�H€EØ€Lr   c                óB   • [        U 5      nU q[        [        5       5      $ )z‰
Register FA4 flash attention kernels with the PyTorch dispatcher.

Args:
    module_path: Python module path to the FA4 implementation.
)Ú_fa4_import_moduler   r   Ú_fa4_register_kernels)Úmodule_pathr,   s     r   r   r   ,   s#   € ô 	˜;Ó'€AØ"ÐÜÔ+Ó-Ó.Ð.r   c                ó”   • [         R                  " U 5      n[        US5      (       a  [        US5      (       d  [        SU  S35      eU$ )NÚ_flash_attn_fwdÚ_flash_attn_bwdzModule 'z' does not expose FA4 kernels)Ú	importlibÚimport_moduleÚhasattrÚRuntimeError)r1   Úmodules     r   r/   r/   ;   sG   € ä×$Ò$ [Ó1€FÜ�6Ð,×-Ñ-´W¸VÐEV×5WÑ5WÜ˜X k ]Ð2OÐPÓQÐQØ€Mr   c                 óØ   • [        SSS5      n U R                  S[        S5        U R                  S[        S5        U R                  S[        S5        U R                  S[
        S5        U $ )NÚatenÚIMPLÚCUDAÚ_flash_attention_forwardÚ_flash_attention_backwardÚ#_scaled_dot_product_flash_attentionÚ,_scaled_dot_product_flash_attention_backward)r   ÚimplÚ!_fa4_flash_attention_forward_implÚ"_fa4_flash_attention_backward_implÚ4_fa4_scaled_dot_product_flash_attention_forward_implÚ5_fa4_scaled_dot_product_flash_attention_backward_impl)Úlibs    r   r0   r0   C   sg   € Ü
�&˜& &Ó
)€CØ‡H�HÐ'Ô)JÈFÔSØ‡H�HÐ(Ô*LÈfÔUØ‡H�HØ-Ü<Øôð
 ‡H�HØ6Ü=Øôð
 €Jr   c                ó,  • [        S U 5       5      (       d  g[        U Vs1 s H  oDR                  iM     sn5      S:w  a  gU R                  [        R
                  [        R                  4;  a  gU H*  u  pVUR                  [        R                  :w  d  M%  U S3s  $    Uc  U R                  5       S:w  a  gUb  U R                  5       S	:w  a  g
[        R                  R                  5       (       d  g[        U R                  5      S;  a  gg s  snf )Nc              3  ó8   #   • U  H  oR                   v •  M     g 7fr   )Úis_cuda©Ú.0Úts     r   Ú	<genexpr>Ú,_fa4_common_support_error.<locals>.<genexpr>Z   s   é € Ð*¢'˜Q�yŽy¢'ùs   ‚zinputs must be CUDA tensorsr
   úinputs must share devicez'query dtype must be float16 or bfloat16z dtype must be float32é   zdense query must be 4Dé   zragged query must be 3DzCUDA not available)é	   é
   z+FA4 requires compute capability 9.0 or 10.0)ÚallÚlenr*   Údtyper'   Úfloat16Úbfloat16Úfloat32Údimr(   Úis_availabler-   )ÚqueryÚtensorsÚ	cum_seq_qÚrequire_fp32rM   ÚnameÚtensors          r   Ú_fa4_common_support_errorrc   T   sß   € ô Ñ*¡'Ó*×*Ñ*Ø,Ü
™gÓ&šg˜�HŒH™gÑ&Ó'¨1Ó,Ø)Ø‡{�{œ5Ÿ=™=¬%¯.©.Ð9Ó9Ø8Û$‰ˆØ�<‰<œ5Ÿ=™=Õ(Ø�VÐ1Ð2Ò2ñ %ð Ñ˜UŸY™Y›[¨AÓ-Ø'ØÑ §¡£°Ó!1Ø(Ü�:‰:×"Ñ"×$Ñ$Ø#Ü˜Ÿ™Ó&¨gÓ5Ø<Øùò 's   ¢Dc                óÈ   • US:w  a  gU(       a  gUb  gUb1  UR                   [        R                  :w  a  gUR                  (       d  g[	        U XU4U5      nUb	  US:X  a  gU$ g )	Nç        údropout_p must be 0zreturn_debug_mask must be Falsezalibi_slopes not supportedzseqused_k must be int32zseqused_k must be CUDArP   z(query, key, value must be on same device)rW   r'   Úint32rJ   rc   )	r]   ÚkeyÚvalueÚ	dropout_pÚreturn_debug_maskÚalibi_slopesÚ	seqused_kr_   Úerrors	            r   Ú_fa4_forward_support_errorro   n   sw   € ð �CÓØ$ÞØ0ØÑØ+ØÑØ�?‰?œeŸk™kÓ)Ø,Ø× × Ø+Ü%ØØ	�UÐØó€Eð
 ÑØÐ.Ó.Ø=ØˆØr   c
           	     óN   • US:w  a  gUc  U	b  g[        UXX#XE4USU44S9n
U
b  U
$ g )Nre   rf   z windowed attention not supportedÚ	logsumexp)r`   )rc   )Úgrad_outr]   rh   ri   Úoutrq   rj   r_   Úwindow_size_leftÚwindow_size_rightrn   s              r   Ú_fa4_backward_support_errorrv   �   sS   € ð �CÓØ$ØÑ#Ð'8Ñ'DØ1Ü%ØØ	˜# cÐ5ØØ" IÐ.Ð0ñ	€Eð ÑØˆØr   ÚTsc                 ó&   • [        S U  5       5      $ )Nc              3  óD   #   • U  H  oR                  S S5      v •  M     g7f)r
   é   N)Ú	transposerK   s     r   rN   Ú#_transpose_dense.<locals>.<genexpr>®   s   é € Ð4ªG q—‘˜Q ×"Ð"ªGùs   ‚ )Útuple)r^   s    r   Ú_transpose_denser~   ­   s   € ÜÑ4©GÓ4Ó4Ð4r   c           	     óÜ   • [         c  [        S5      e[        [         5      nUUUUSUUU	b  U	R                  5       OS S.nU
b  X¬S'   UR                  " XU40 UD6u  p­X­R                  5       4$ )NúFA4 not registeredT)Úsoftmax_scaleÚcausalrt   ru   Ú
return_lseÚcu_seqlens_qÚcu_seqlens_krm   rs   )r   r8   r/   Ú
contiguousr3   )r]   rh   ri   Úcu_seq_qÚcu_seq_kÚscaleÚ	is_causalrt   ru   rm   rs   r9   ÚkwargsÚlses                 r   Ú_fa4_run_forwardr�   ±   s‰   € ô ÑÜÐ/Ó0Ð0ÜÔ 0Ó1€Fð ØØ,Ø.ØØ Ø Ø/8Ñ/D�Y×)Ñ)Ô+È$ñ	€Fð �Øˆu‰Ø×%Ò% e°%ÑB¸6ÑB�H€CØ—‘Ó Ð Ð r   c                ó    • [         c  [        S5      e[        [         5      nUR                  UUUUU UR	                  5       UU	UUU
S9u  pÍnXÍU4$ )Nr€   )r�   r‚   r„   r…   Údeterministic)r   r8   r/   r4   r†   )rr   r]   rh   ri   rs   rq   r‡   rˆ   r‰   rŠ   r�   r9   ÚdqÚdkÚdvs                  r   Ú_fa4_run_backwardr“   Ò   sp   € ô ÑÜÐ/Ó0Ð0ÜÔ 0Ó1€FØ×'Ñ'ØØØØØØ×ÑÓØØØØØ#ð (ð �J€BˆBð �2ˆ:Ðr   )r‰   rt   ru   rm   rl   rs   c
               ó’  • [        U UUUU	UUU5      nUb  [        SU 35      e[        U UUUUU
UUUUU5      u  nn[        R                  " S[        R
                  U R                  S9n[        R                  " S[        R
                  U R                  S9n[        R                  " SU R                  U R                  S9nUUUUU4$ )Nz)FA4 flash_attention forward unsupported: )rz   )rW   r*   r%   r   )	ro   r8   r�   r'   ÚzerosÚuint64r*   ÚemptyrW   )r]   rh   ri   r_   Ú	cum_seq_kÚmax_qÚmax_krj   rŠ   rk   r‰   rt   ru   rm   rl   rs   rn   rŒ   Ú	rng_stateÚphilox_offsetÚ
debug_masks                        r   rC   rC   ò   sÑ   € ô& 'ØØØØØØØØó	€Eð ÑÜÐFÀuÀgÐNÓOÐOÜØØØØØØØØØØØó�H€Cˆô —’˜D¬¯©¸U¿\¹\ÑJ€IÜ—K’K ¬%¯,©,¸u¿|¹|ÑL€MÜ—’˜Q e§k¡k¸%¿,¹,ÑG€JØ��Y ¨zÐ9Ð9r   )r‰   rt   ru   c               ó²   • [        U UUUUUU
UUU5
      nUb  [        SU 35      e[        R                  " 5       n[	        U UUUUUUUUUU5      u  nnnUUU4$ )Nz*FA4 flash_attention backward unsupported: )rv   r8   r'   Ú$are_deterministic_algorithms_enabledr“   )rr   r]   rh   ri   rs   rq   r_   r˜   r™   rš   rj   rŠ   r›   Úunusedr‰   rt   ru   rn   r�   r�   r‘   r’   s                         r   rD   rD   $  s”   € ô( (ØØØØØØØØØØó€Eð ÑÜÐGÈÀwÐOÓPÐPÜ×>Ò>Ó@€MÜ"ØØØØØØØØØØØó�J€BˆˆBð ˆr�2ˆ:Ðr   ©r‰   c               óˆ  • [        U UUUUS S S 5      nUb  [        SU 35      e[        XU5      u  p‰n
[        R                  " U 5      nUR                  SS5      nUR                  S5      nU	R                  S5      n[        UU	U
S S UUUUUUUS9u  nnnnnU R                  S5      nUR                  S5      nUUS S UUUUU4	$ )NzFA4 SDPA forward unsupported: r
   rz   )r‰   rs   )ro   r8   r~   r'   Ú
empty_liker{   ÚsizerC   )r]   rh   ri   rj   rŠ   rk   r‰   rn   ÚqÚkÚvÚout_bhsdÚout_bshdÚmax_q_flashÚmax_k_flashr,   rŒ   r›   rœ   r�   r™   rš   s                         r   rE   rE   W  s	  € ô 'ØØØØØØØØó	€Eð ÑÜÐ;¸E¸7ÐCÓDÐDÜ˜u¨5Ó1�G€Aˆ!ô
 ×Ò Ó&€HØ×!Ñ! ! QÓ'€Hà—&‘&˜“)€KØ—&‘&˜“)€KÜ3TØ	Ø	Ø	ØØØØØØØØØñ4Ñ0€A€sˆI�} jð �J‰J�q‹M€EØ�H‰H�Q‹K€EàØØØØØØØØð
ð 
r   c               ó  • [        U UUUUUU
S S S 5
      nUb  [        SU 35      e[        XX4U 5      u  nnnnnUR                  S5      nUR                  S5      n	[	        UUUUUUS S UU	U
UUUUS9u  nnn[        UUU5      u  nnnUUU4$ )NzFA4 SDPA backward unsupported: rz   r¡   )rv   r8   r~   r¤   rD   )rr   r]   rh   ri   rs   rq   r_   r˜   r™   rš   rj   rŠ   Úphilox_seedrœ   r‰   rn   r¥   r¦   r§   ÚoÚgor�   r‘   r’   s                           r   rF   rF   ”  sØ   € ô$ (ØØØØØØØØØØó€Eð ÑÜÐ<¸U¸GÐDÓEÐEÜ% e°%¸hÓG�N€A€qˆ!ˆQ�Ø�J‰J�q‹M€EØ�H‰H�Q‹K€EÜ3Ø
Ø	Ø	Ø	Ø	ØØØØØØØØØØñ�J€BˆˆBô" " " b¨"Ó-�J€BˆˆBØˆr�2ˆ:Ðr   ÚFA4)Úregister_fn)r*   ztorch.devicer   Úint)zflash_attn.cute.interface)r1   Ústrr   r   )r1   r³   r   r   )r   r   )r%   )
r]   útorch.Tensorr^   ztuple[torch.Tensor, ...]r_   útorch.Tensor | Noner`   z$tuple[tuple[str, torch.Tensor], ...]r   r   )r]   r´   rh   r´   ri   r´   rj   Úfloatrk   Úboolrl   rµ   rm   rµ   r_   rµ   r   r   )rr   r´   r]   r´   rh   r´   ri   r´   rs   r´   rq   r´   rj   r¶   r_   rµ   rt   ú
int | Noneru   r¸   r   r   )r^   z
Unpack[Ts]r   ztuple[Unpack[Ts]]r   )r]   r´   rh   r´   ri   r´   r‡   rµ   rˆ   rµ   r‰   úfloat | NonerŠ   r·   rt   r¸   ru   r¸   rm   rµ   rs   rµ   r   z!tuple[torch.Tensor, torch.Tensor])F)rr   r´   r]   r´   rh   r´   ri   r´   rs   r´   rq   r´   r‡   rµ   rˆ   rµ   r‰   r¹   rŠ   r·   r�   r·   r   z/tuple[torch.Tensor, torch.Tensor, torch.Tensor]) r]   r´   rh   r´   ri   r´   r_   rµ   r˜   rµ   r™   r²   rš   r²   rj   r¶   rŠ   r·   rk   r·   r‰   r¹   rt   r¸   ru   r¸   rm   rµ   rl   rµ   rs   rµ   )"rr   r´   r]   r´   rh   r´   ri   r´   rs   r´   rq   r´   r_   rµ   r˜   rµ   r™   r²   rš   r²   rj   r¶   rŠ   r·   r›   r´   r    r´   r‰   r¹   rt   r¸   ru   r¸   )re   FF)r]   r´   rh   r´   ri   r´   rj   r¶   rŠ   r·   rk   r·   r‰   r¹   )rr   r´   r]   r´   rh   r´   ri   r´   rs   r´   rq   r´   r_   rµ   r˜   rµ   r™   r²   rš   r²   rj   r¶   rŠ   r·   r­   r´   rœ   r´   r‰   r¹   ))Ú__doc__Ú
__future__r   r5   Údataclassesr   Ú	functoolsr   Útypingr   r   Útyping_extensionsr   r	   Ú r   Útypesr   r'   Útorch.libraryr   Ú__all__r   r#   r   r-   r   r/   r0   rc   ro   rv   rw   r~   r�   r“   rC   rD   rE   rF   Úregister_flash_attention_implr%   r   r   Ú<module>rÅ      s2  ðÚ õ #ã Ý !Ý ß %ß 2å ö Ý ã Ý !ð #ð€ð
  $Ð �*Ó #ð ÷ð ó ðð óó ðð 3ð/Øð/àõ/ð óó ðôð* :<ð	Øðà%ðð #ðð 7ð	ð
 õð4Øðà	ðð ðð ð	ð
 ðð &ðð #ðð #ðð ôðBØðàðð 
ðð ð	ð
 
ðð ðð ðð #ðð !ðð "ðð ôñ6 �$Ó€ô5ð  $ð!Øð!à	ð!ð ð!ð "ð	!ð
 "ð!ð ð!ð ð!ð !ð!ð "ð!ð #ð!ð 
ð!ð 'õ!ðX  ðØðàðð 
ðð ð	ð
 
ðð ðð "ðð "ðð ðð ðð ðð 5õðX Ø#'Ø$(Ø%)Ø(,Ø#ñ#/:Øð/:à	ð/:ð ð/:ð #ð	/:ð
 #ð/:ð ð/:ð ð/:ð ð/:ð ð/:ð ð/:ð ð/:ð !ð/:ð "ð/:ð #ð/:ð  &ð!/:ð" 
õ#/:ðD Ø#'Ø$(ñ%0Øð0àð0ð 
ð0ð ð	0ð
 
ð0ð ð0ð #ð0ð #ð0ð ð0ð ð0ð ð0ð ð0ð ð0ð ð0ð  ð!0ð" !ð#0ð$ "õ%0ðn ØØ#ð:ð ñ:Øð:à	ð:ð ð:ð ð	:ð
 ð:ð ð:ð ö:ðZ ñ!5Øð5àð5ð 
ð5ð ð	5ð
 
ð5ð ð5ð #ð5ð #ð5ð ð5ð ð5ð ð5ð ð5ð ð5ð  ð5ð  õ!5ðp 
× 'Ò '¨Ð;WÓ Xr   