ó
    !EñiÖ ã                   óæ  • % S r SSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKJ	r	  SSK
Jr  SSKJrJrJrJr  SSKJrJr  SSKrSSKJr  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJrJ r   Sr!\"" 5       r#\"\$   \%S'   \&4S\$S\$S\'\(   SS4S jjr)/ SQr*\	\\\\\/\4   r+\	\\\\/\4   r,\S   r-\\%S'    " S S\SS9r. " S S\5      r/ " S S\5      r0 " S S\5      r1S \	S\14S! jr2   SiS \	S"\3\4S-  S#4   S$\3\4S-  S#4   S%\4\5\4S-     -  S&\64
S' jjr7S(\S)\S*\S+\S,\S\4S- jr8S)\S*\S+\S,\S\4
S. jr9S)\S*\S+\S,\S\4
S/ jr:S0r;S1r<S2\S3\4S4 jr=S\3\\4   4S5 jr>S2\S3\4S6 jr?S7\S8\S9\4S:\44S; jr@S \AS\3\AS#4   4S< jrB " S= S>5      rC " S? S@5      rDSA rESB rF\;\;S4SC\SD\6S\3\\S-  4   4SE jjrGSF\,S\,4SG jrHSF\,S\,4SH jrI\;\;4S\4SI jjrJ\;\;4SJ\3\\S-  4   SK\	S-  SL\3\4\44   SM\4SN\4S\D4SO jjrK SjSP\+\,-  SQ\4S-  SR\4S-  SS\4ST\4SU\S-  S\4SV jjrLS\;S4SK\,SQ\4S-  SR\4S-  SS\4ST\4SU\S-  SW\4\3\4\44   -  S\D4SX jjrMSY\SZ\S\D4S[ jrN SjSY\SZ\S\\S]\6S^\/S-  4
S_ jjrOSY\SZ\S\\SS4S` jrPSY\SZ\S\\SS4Sa jrQSY\SZ\S\\S\3\R(                  \R(                  \R(                  4   4Sb jrR      SkSSc.SY\SZ\S\\Sd\+S-  SJ\DS-  Se\SS-  Sf\6S]\6Sg\.S-  S^\/S-  S\\3\\4   -  \3\\04   -  4Sh jjjrg)lzIThis module implements the user facing API for flex_attention in PyTorch.é    N)ÚCallable)ÚEnum)ÚAnyÚLiteralÚ
NamedTupleÚ	TypeAlias)ÚNotRequiredÚ	TypedDict)ÚTensor)Úflex_attention)Úsetup_compilation_env)ÚDeviceLikeType)Ú_validate_sdpa_input)Ú
GetAttrKeyÚtree_map_onlyFÚ_WARNINGS_SHOWNÚ
warning_idÚmessageÚcategoryÚreturnc                 ó´   • U [         ;  aN  [        R                  R                  5       (       d  [        R
                  " XSS9  [         R                  U 5        gg)z=Helper to ensure each warning is shown only once per process.é   ©Ú
stacklevelN)r   ÚtorchÚcompilerÚis_compilingÚwarningsÚwarnÚadd)r   r   r   s      Ú^/home/mande/repo/quber/.venv/lib/python3.13/site-packages/torch/nn/attention/flex_attention.pyÚ
_warn_oncer"   /   s@   € ð œÓ(Ü�~‰~×*Ñ*×,Ñ,Ü�MŠM˜'¸Ò:Ü×Ñ˜JÕ'ð )ó    )
Ú	BlockMaskr   Ú	AuxOutputÚ
AuxRequestÚFlexKernelOptionsÚcreate_block_maskÚcreate_maskÚor_masksÚ	and_masksÚ	noop_mask)ÚAUTOÚTRITONÚFLASHÚTRITON_DECODEÚ_Backendc                   ó\  • \ rS rSr% Sr\\   \S'    \\   \S'    \\   \S'    \\   \S'    \\   \S'    \\   \S'    \\   \S	'    \\   \S
'    \\   \S'    \\   \S'    \\   \S'    \\   \S'    \\   \S'    \\   \S'    \\   \S'    \\   \S'    \\   \S'    \\	   \S'   Sr
g)r'   éK   ab  Options for controlling the behavior of FlexAttention kernels.

These options are passed to the underlying Triton kernels to control performance
and numerical behavior. Most users will not need to specify these options as the
default autotuning provides good performance.

The options can be prefixed with ``fwd_`` or ``bwd_`` to apply only to forward or
backward pass respectively. For example: ``fwd_BLOCK_M`` and ``bwd_BLOCK_M1``.

Note:
  We currently do not provide any backward compatibility guarantees for these options.
  That being said most of these have remained pretty stable since their introduction. But
  We do not consider this part of the public API just yet. We think that some documentation
  Is better than secret hidden flags, but we may change these options in the future.

Example Usage:
    .. code-block:: python

        # Using dictionary (backward compatible)
        kernel_opts = {"BLOCK_M": 64, "BLOCK_N": 64, "PRESCALE_QK": True}
        output = flex_attention(q, k, v, kernel_options=kernel_opts)

        # Using TypedDict (recommended for type safety)
        from torch.nn.attention.flex_attention import FlexKernelOptions

        kernel_opts: FlexKernelOptions = {
            "BLOCK_M": 64,
            "BLOCK_N": 64,
            "PRESCALE_QK": True,
        }
        output = flex_attention(q, k, v, kernel_options=kernel_opts)

        # Forward/backward specific options
        kernel_opts: FlexKernelOptions = {
            "fwd_BLOCK_M": 64,
            "bwd_BLOCK_M1": 32,
            "PRESCALE_QK": False,
        }
        output = flex_attention(q, k, v, kernel_options=kernel_opts)
Ú	num_warpsÚ
num_stagesÚBLOCK_MÚBLOCK_NÚBLOCK_M1ÚBLOCK_N1ÚBLOCK_M2ÚBLOCK_N2ÚPRESCALE_QKÚROWS_GUARANTEED_SAFEÚBLOCKS_ARE_CONTIGUOUSÚWRITE_DQÚFORCE_USE_FLEX_ATTENTIONÚUSE_TMAÚkpackÚmatrix_instr_nonkdimÚwaves_per_euÚBACKEND© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r	   ÚintÚ__annotations__Úboolr1   Ú__static_attributes__rF   r#   r!   r'   r'   K   sG  ‡ ñ'ðV ˜3ÑÓðLð ˜CÑ Ó ðNð ˜ÑÓðbð ˜ÑÓðbð
 ˜#ÑÓð,ð ˜#ÑÓð,ð ˜#ÑÓð,ð ˜#ÑÓð,ð ˜TÑ"Ó"ð6ð & dÑ+Ó+ðQð
 ' tÑ,Ó,ðAð ˜$ÑÓðGð *¨$Ñ/Ó/ð-ð ˜ÑÓð/ð �sÑÓØ1à% cÑ*Ó*Ø;à˜cÑ"Ó"Ø1à˜Ñ"Ó"òr#   r'   )Útotalc                   ó6   • \ rS rSr% SrSr\\S'   Sr\\S'   Sr	g)r&   éÑ   z–Request which auxiliary outputs to compute from flex_attention.

Each field is a boolean indicating whether that auxiliary output should be computed.
FÚlseÚ
max_scoresrF   N)
rG   rH   rI   rJ   rK   rS   rN   rM   rT   rO   rF   r#   r!   r&   r&   Ñ   s   ‡ ñð
 €CˆÓØ€J�Ör#   r&   c                   óB   • \ rS rSr% SrSr\S-  \S'   Sr\S-  \S'   Sr	g)r%   éÛ   z|Auxiliary outputs from flex_attention operation.

Fields will be None if not requested, or contain the tensor if requested.
NrS   rT   rF   )
rG   rH   rI   rJ   rK   rS   r   rM   rT   rO   rF   r#   r!   r%   r%   Û   s%   ‡ ñð
 €Cˆ�$‰ÓØ $€J�˜‘Ö$r#   r%   c                   ó$   • \ rS rSrSrSrSrSrSrg)Ú_ModificationTypeéå   zÛEnum for the type of modification function.
- SCORE_MOD: score_mod function which accepts a score as the first argument
- mask_mod: mask function which does not accept a score and is only used for generating
block mask
é   r   é   rF   N)	rG   rH   rI   rJ   rK   Ú	SCORE_MODÚMASK_MODÚUNKNOWNrO   rF   r#   r!   rX   rX   å   s   † ñð €IØ€HØƒGr#   rX   Úfnc                 óì  • [        U S5      (       aP  U R                  nUR                  nSn[        U S5      (       a  U R                  =(       d    Sn[	        U5      nX$-
  nO>[        S [        R                  " U 5      R                  R                  5        5       5      nUS:w  a  US:w  a  [        SU 35      eUS:X  a  [        R                  $ US:X  a  [        R                  $ [        R                  $ )aH  Get the type of modification function.
This function inspects the number of positional arguments of the function to determine
the type of modification function. If the function has 5 positional arguments, it is
considered as a score_mod function. If the function has 4 positional arguments, it is
considered as a mask function.
Ú__code__rF   Ú__defaults__c              3   óx   #   • U  H0  nUR                   [        R                  R                  L d  M,  S v •  M2     g7f)rZ   N)ÚdefaultÚinspectÚ	ParameterÚempty)Ú.0Úparams     r!   Ú	<genexpr>Ú _get_mod_type.<locals>.<genexpr>  s1   é € ð "
âB�Ø�}‰}¤× 1Ñ 1× 7Ñ 7Ð7÷ ‰AÚBùs   ‚+:±	:é   é   z%Expected 4 or 5 positional args, got )Úhasattrra   Úco_argcountrb   ÚlenÚsumre   Ú	signatureÚ
parametersÚvaluesÚAssertionErrorrX   r\   r]   r^   )r_   ÚcodeÚnum_positional_totalÚdefaultsÚnum_defaultsÚnum_positional_argss         r!   Ú_get_mod_typer{   ñ   së   € ô ˆr�:×ÑØ�{‰{ˆØ#×/Ñ/ÐØˆÜ�2�~×&Ñ&Ø—‘×,¨"ˆHÜ˜8“}ˆØ2ÑAÑä!ñ "
ä ×*Ò*¨2Ó.×9Ñ9×@Ñ@ÔBó"
ó 
Ðð
 ˜aÓÐ$7¸1Ó$<ÜØ3Ð4GÐ3HÐIó
ð 	
ð ˜aÓÜ ×*Ñ*Ð*Ø	 Ó	!Ü ×)Ñ)Ð)ä ×(Ñ(Ð(r#   Úprefix.ÚsuffixÚout_dimsÚ	group_dimc                 ó€   • / n/ SQnU(       a  US/-  nUS/-  nU H  n[         R                  " XU-   U-   US9n M      U $ )a[  Used to vmap both score_mods and mask_mods over 4-dimensional/5-dimension inputs.
Mapping over the [b, hq, q_idx, kv_idx] or [b, hkv, g, q_idx, kv_idx] dimensions.

Args:
    fn (callable): The function to vmap.
    prefix (tuple): The prefix of the vmap. For score mod functions,
                    this should be set to (0,). For mask_mods = ()
    suffix (tuple): We need to add (0,) if gradOut is being mapped over,
                    and (None,) * len(other_buffers).
    out_dims (tuple): For forward cases, keep this as the default 0 since
                      we are only returning 1 output. For backwards, the joint
                      graph returns grads for B, H, Q_idx, KV_idx and other_buffers,
                      so we set this to (0, None, None, None, None) + (None,) * len(other_buffers).

Returns:
    callable: The vmapped function.
))NNNr   )NNr   N©Nr   NNr�   )r   NNN)Úin_dimsr~   )r   Úvmap)r_   r|   r}   r~   r   Ú
dimensionsÚdimss          r!   Ú_vmap_for_bhqkvr†     sg   € ð2 OQ€Jò€Jö ØØ!ð
ñ 	
ˆ
ð Øðñ €Jó ˆÜ�ZŠZ˜¨T¡M°FÑ$:ÀXÑNŠñ à€Ir#   ÚscoreÚbatchÚheadÚtoken_qÚtoken_kvc                 ó   • U $ ©NrF   )r‡   rˆ   r‰   rŠ   r‹   s        r!   Ú	_identityrŽ   A  s	   € ð €Lr#   c                 óT   • U R                  S[        R                  U R                  S9$ )zReturns a noop mask_modrF   )ÚsizeÚdtypeÚdevice)Únew_onesr   rN   r’   ©rˆ   r‰   rŠ   r‹   s       r!   r,   r,   K  s!   € ð �>‰>˜r¬¯©¸E¿L¹Lˆ>ÐIÐIr#   c                 ó   • [        S5      e)z×
Raises helpful error when using mask_mod from a sliced BlockMask.

After slicing a BlockMask, the mask_mod is reset and cannot be used directly.
Users must reassign mask_mod from the original (unsliced) BlockMask.
a&  Cannot use mask_mod from a sliced BlockMask. When you slice a BlockMask using [], the mask_mod attribute is reset. You must set it from the original BlockMask's mask_mod.

Incorrect usage:
  base_mask = create_block_mask(my_mask_fn, ...)
  sliced_mask = base_mask[:, :, block_idx]
  sliced_mask.mask_mod = apply_offset(sliced_mask.mask_mod, offset)  # WRONG!

Correct usage:
  base_mask = create_block_mask(my_mask_fn, ...)
  sliced_mask = base_mask[:, :, block_idx]
  sliced_mask.mask_mod = apply_offset(base_mask.mask_mod, offset)  # Use base_mask!)ÚRuntimeErrorr”   s       r!   Ú_sliced_mask_mod_errorr—   U  s   € ô ð
	`óð r#   é€   i   @Únum_blocks_in_rowÚcol_indicesc                 ó  ^^^	• UR                   S   m	UR                   S   mU R                   S S nU R                  mUUU	4S jnUn[        [        U5      5       H  n[        R
                  " USS9nM     U" X5      nU$ )Néþÿÿÿéÿÿÿÿc                 ó¦  >• UR                  T	TS-   [        R                  S9n[        R                  " T	[        R                  TS9R                  S5      n[        R                  " T[        R                  TS9nX@R                  S5      :  n[        R                  " XQT5      nUR                  S5      X#U4'   US S 2S T24   R                  5       $ )NrZ   ©r‘   ©r‘   r’   r�   rF   )	Ú	new_zerosr   Úint32ÚarangerL   Ú	unsqueezeÚwherer“   Ú
contiguous)
Úkv_num_blocksÚ
kv_indicesÚ
dense_maskÚrow_indicesÚ	col_rangeÚ
index_maskÚvalid_indicesr’   Únum_colsÚnum_rowss
          €€€r!   Úcreate_dense_oneÚ+_ordered_to_dense.<locals>.create_dense_onez  s¹   ø€ Ø×)Ñ)¨(°H¸q±LÌÏÉÐ)ÐTˆ
ä—l’l 8´5·9±9ÀVÑL×VÑVØó
ˆô —L’L ´·±À6ÑJˆ	Ø×!8Ñ!8¸Ó!<Ñ<ˆ
ô Ÿš J¸HÓEˆð 2<×1DÑ1DÀRÓ1Hˆ
 Ð-Ñ.Øš!˜Y˜h˜Y˜,Ñ'×2Ñ2Ó4Ð4r#   )r   r   )r‚   )Úshaper’   Úrangerp   r   rƒ   )
r™   rš   Ú
batch_dimsr°   Úcreate_dense_batchedÚ_Úoutr’   r®   r¯   s
          @@@r!   Ú_ordered_to_denser¸   t  s„   ú€ Ø× Ñ  Ñ$€HØ× Ñ  Ñ$€HØ"×(Ñ(¨¨"Ð-€JØ×%Ñ%€F÷5ð  ,ÐÜ”3�z“?Ö#ˆÜ$ŸzšzÐ*>ÈÑOÒñ $ñ Ð0Ó
>€CØ€Jr#   c                 ó8  • U R                  [        R                  S9n U R                  SS9n[        R                  " U SSSS9nUR                  [        R                  [        R
                  S9UR                  [        R                  [        R
                  S94$ )NrŸ   r�   ©ÚdimT)r»   Ú
descendingÚstable)Úmemory_format)Útor   r¢   rq   ÚargsortÚcontiguous_format)r©   r™   rš   s      r!   Ú_dense_to_orderedrÂ   ’  s|   € Ø—‘¤U§[¡[�Ð1€JØ"Ÿ™¨2˜Ð.ÐÜ—-’- 
°¸tÈDÑQ€Kà×ÑœUŸ[™[¼×8OÑ8OÐÐPØ�‰”u—{‘{´%×2IÑ2IˆÐJðð r#   c                 óN   • [        X5      n[        UR                  SS5      5      $ )Nrœ   r�   )r¸   rÂ   Ú	transpose)r™   rš   Údenses      r!   Ú_transpose_orderedrÆ   œ  s#   € ÜÐ/Ó=€EÜ˜UŸ_™_¨R°Ó4Ó5Ð5r#   Ú
num_blocksÚindicesÚnew_num_rowsÚnew_num_colsc                 óú   • US S 2S S 2S U2S U24   nU S S 2S S 2S U24   n [         R                  " X:  X5      n [         R                  " XS S 2S S 2S S 2S 4   :  SS9R                  [         R                  5      n X4$ )Nr�   rº   )r   r¥   rq   r¿   r¢   )rÇ   rÈ   rÉ   rÊ   s       r!   Ú_adjust_num_blocks_and_indicesrÌ   ¡  s   € ð ’aš˜M˜\˜M¨=¨L¨=Ð8Ñ9€GØšAšq - < -Ð/Ñ0€JÜ—’˜ZÑ6¸
ÓQ€JÜ—’˜7²²1²a¸°Ñ%>Ñ>ÀBÑG×JÑJÌ5Ï;É;ÓW€JØÐÐr#   c                 óH   • [        U SS5      nUc  g[        S U 5       5      $ )z-Extract closure cell contents for comparison.Ú__closure__NrF   c              3   ó8   #   • U  H  oR                   v •  M     g 7fr�   )Úcell_contents)rh   Úcells     r!   rj   Ú$_closure_contents.<locals>.<genexpr>³  s   é € Ð8²¨×#Ö#²ùó   ‚)ÚgetattrÚtuple)r_   Úclosures     r!   Ú_closure_contentsr×   ®  s)   € ä�b˜-¨Ó.€GØ�ØÜÑ8±Ó8Ó8Ð8r#   c                   ób   • \ rS rSrSrSrS\SS4S jrS rS	\	S\
4S
 jrS\4S jrS\4S jrSrg)Ú_MaskModWrapperi¶  a^  Wraps a mask_mod function with value-based equality.

BlockMask stores an arbitrary callable (mask_mod) in its pytree context.
The default __eq__ for functions uses identity comparison, which is too
strict when the same closure is recreated (e.g., defined inside forward()).
This wrapper compares functions by their code object and closure contents.
©r_   r_   r   Nc                 ó   • Xl         g r�   rÚ   )Úselfr_   s     r!   Ú__init__Ú_MaskModWrapper.__init__Á  s   € Ø�r#   c                 ó&   • U R                   " U0 UD6$ r�   rÚ   )rÜ   ÚargsÚkwargss      r!   Ú__call__Ú_MaskModWrapper.__call__Ä  s   € Ø�wŠw˜Ð' Ñ'Ð'r#   Úotherc                 ó®  • [        U[        5      (       d  gU R                  UR                  L a  g[        R                  " U R                  5      (       a€  [        R                  " UR                  5      (       a[  U R                  R
                  UR                  R
                  :X  a-  [        U R                  5      [        UR                  5      :X  a  gg)NFT)Ú
isinstancerÙ   r_   re   Ú
isfunctionra   r×   )rÜ   rä   s     r!   Ú__eq__Ú_MaskModWrapper.__eq__Ç  sŒ   € Ü˜%¤×1Ñ1ØØ�7‰7�e—h‘hÒØä×Ò˜tŸw™w×'Ñ'Ü×"Ò" 5§8¡8×,Ñ,Ø—‘× Ñ  E§H¡H×$5Ñ$5Ó5Ü! $§'¡'Ó*Ô.?ÀÇÁÓ.IÓIàØr#   c                 ó´   • [         R                  " U R                  5      (       a  [        U R                  R                  5      $ [        U R                  5      $ r�   )re   rç   r_   Úhashra   ©rÜ   s    r!   Ú__hash__Ú_MaskModWrapper.__hash__Õ  s:   € Ü×Ò˜dŸg™g×&Ñ&Ü˜Ÿ™×(Ñ(Ó)Ð)Ü�D—G‘G‹}Ðr#   c                 ó"   • SU R                    S3$ )Nz_MaskModWrapper(Ú)rÚ   rì   s    r!   Ú__repr__Ú_MaskModWrapper.__repr__Ú  s   € Ø! $§'¡' ¨!Ð,Ð,r#   )rG   rH   rI   rJ   rK   Ú	__slots__Ú_mask_mod_signaturerÝ   râ   ÚobjectrN   rè   rL   rí   Ústrrñ   rO   rF   r#   r!   rÙ   rÙ   ¶  sR   † ñð €IðÐ.ð °4ô ò(ð˜Fð  tô ð˜#ô ð
-˜#÷ -r#   rÙ   c                   óÆ  • \ rS rSr% Sr\\\4   \S'   \\S'   \\S'   \S-  \S'   \S-  \S'   \S-  \S	'   \S-  \S
'   \S-  \S'   \S-  \S'   \\\4   \S'   \	\S'   / SQr
/ SQrS\\\4   S\S\S\S-  S\S-  S	\S-  S
\S-  S\S-  S\S-  S\\\4   S\	SS4S jr\SS\SSS4S\S\S\S-  S\S-  S\\\\4   -  S\	S-  S\\\4   S-  S\4S jj5       rS-S\4S jjr\S 5       rS\4S jrS.S jrS\4S jrS\S\4S jrS rS\4S  jrS\4S! jrS/S" jrS#\R>                  \-  SS 4S$ jr \!S%\S&\"S\"4S' j5       r#\!S%\S&\"S\"4S( j5       r$S) r%\S* 5       r&S+ r'S,r(g)0r$   iÞ  a‡	  
BlockMask is our format for representing a block-sparse attention mask.
It is somewhat of a cross in-between BCSR and a non-sparse format.

**Basics**

A block-sparse mask means that instead of representing the sparsity of
individual elements in the mask, a KV_BLOCK_SIZE x Q_BLOCK_SIZE block is
considered sparse only if every element within that block is sparse.
This aligns well with hardware, which generally expects to perform
contiguous loads and computation.

This format is primarily optimized for 1. simplicity, and 2. kernel
efficiency. Notably, it is *not* optimized for size, as this mask is always
reduced by a factor of KV_BLOCK_SIZE * Q_BLOCK_SIZE. If the size is a
concern, the tensors can be reduced in size by increasing the block size.

The essentials of our format are:

num_blocks_in_row: Tensor[ROWS]:
Describes the number of blocks present in each row.

col_indices: Tensor[ROWS, MAX_BLOCKS_IN_COL]:
`col_indices[i]` is the sequence of block positions for row i. The values of
this row after `col_indices[i][num_blocks_in_row[i]]` are undefined.

For example, to reconstruct the original tensor from this format:

.. code-block:: python

    dense_mask = torch.zeros(ROWS, COLS)
    for row in range(ROWS):
        for block_idx in range(num_blocks_in_row[row]):
            dense_mask[row, col_indices[row, block_idx]] = 1

Notably, this format makes it easier to implement a reduction along the
*rows* of the mask.

**Details**

The basics of our format require only kv_num_blocks and kv_indices. But, we
have up to 8 tensors on this object. This represents 4 pairs:

1. (kv_num_blocks, kv_indices): Used for the forwards pass of attention, as
we reduce along the KV dimension.

2. [OPTIONAL] (full_kv_num_blocks, full_kv_indices): This is optional and
purely an optimization. As it turns out, applying masking to every block
is quite expensive! If we specifically know which blocks are "full" and
don't require masking at all, then we can skip applying mask_mod to these
blocks. This requires the user to split out a separate mask_mod from the
score_mod. For causal masks, this is about a 15% speedup.

3. [GENERATED] (q_num_blocks, q_indices): Required for the backwards pass,
as computing dKV requires iterating along the mask along the Q dimension. These are autogenerated from 1.

4. [GENERATED] (full_q_num_blocks, full_q_indices): Same as above, but for
the backwards pass. These are autogenerated from 2.
Úseq_lengthsr§   r¨   NÚfull_kv_num_blocksÚfull_kv_indicesÚq_num_blocksÚ	q_indicesÚfull_q_num_blocksÚfull_q_indicesÚ
BLOCK_SIZEÚmask_mod)r§   r¨   rù   rú   rû   rü   rý   rþ   )rø   rÿ   r   r   c                 óR  • UR                  5       S:  a  [        S5      eUc  [        S5      eUc  [        S5      eUS L US L :w  a  [        S5      eUS L U	S L :w  a  [        S5      eXl        X l        X0l        X@l        XPl        X`l        Xpl	        X€l
        X�l        X l        X°l        g )Nr   ú)BlockMask must have at least 2 dimensionszkv_num_blocks must be providedzkv_indices must be providedúGfull_kv_num_blocks and full_kv_indices must be both provided or omittedzEfull_q_num_blocks and full_q_indices must be both provided or omitted)r»   r–   ru   rø   r§   r¨   rù   rú   rû   rü   rý   rþ   rÿ   r   )rÜ   rø   r§   r¨   rù   rú   rû   rü   rý   rþ   rÿ   r   s               r!   rÝ   ÚBlockMask.__init__9  sÉ   € ð �>‰>Ó˜aÓÜÐJÓKÐKØÑ Ü Ð!AÓBÐBØÑÜ Ð!>Ó?Ð?Ø $Ð&¨O¸tÐ,CÓDÜ ØYóð ð  Ð%¨>¸TÐ+AÓBÜ ØWóð ð 'ÔØ*ÔØ$ŒØ"4ÔØ.ÔØ(ÔØ"ŒØ!2ÔØ,ÔØ$ŒØ �r#   TÚcompute_q_blocksc	                 ó°  • UR                  5       S:  a  [        S5      eUSL USL :w  a  [        S5      eU(       a1  [        X5      u  pšUb  Uc  [        S5      e[        X45      u  p¼OSu  p¼OSu  pšSu  p¼[	        U[
        5      (       a  XU4nUb  UO[        nUc-  UR                  S   US   -  nUR                  S	   US
   -  nXÞ4nU " UUUUUU	U
UUUUS9$ )a'  
Creates a BlockMask instance from key-value block information.

Args:
    kv_num_blocks (Tensor): Number of kv_blocks in each Q_BLOCK_SIZE row tile.
    kv_indices (Tensor): Indices of key-value blocks in each Q_BLOCK_SIZE row tile.
    full_kv_num_blocks (Optional[Tensor]): Number of full kv_blocks in each Q_BLOCK_SIZE row tile.
    full_kv_indices (Optional[Tensor]): Indices of full key-value blocks in each Q_BLOCK_SIZE row tile.
    BLOCK_SIZE (Union[int, tuple[int, int]]): Size of KV_BLOCK_SIZE x Q_BLOCK_SIZE tiles.
    mask_mod (Optional[Callable]): Function to modify the mask.

Returns:
    BlockMask: Instance with full Q information generated via _transposed_ordered

Raises:
    RuntimeError: If kv_indices has < 2 dimensions.
    AssertionError: If only one of full_kv_* args is provided.
r   r  Nr  ú full_kv_indices must not be None©NNrœ   r   r�   rZ   )rø   r§   r¨   rù   rú   rû   rü   rý   rþ   rÿ   r   )r»   r–   ru   rÆ   ræ   rL   r,   r²   )Úclsr§   r¨   rù   rú   rÿ   r   rø   r  rû   rü   rý   rþ   Úq_lengthÚ	kv_lengths                  r!   Úfrom_kv_blocksÚBlockMask.from_kv_blocksb  s+  € ð< �>‰>Ó˜aÓÜÐJÓKÐKà $Ð&¨O¸tÐ,CÓDÜ ØYóð ö
 Ü&8¸Ó&SÑ#ˆLØ!Ñ-Ø"Ñ*Ü(Ð)KÓLÐLÜ4FØ&ó5Ñ1Ð! >ð 5?Ñ1Ð! >à&0Ñ#ˆLØ0:Ñ-Ðä�j¤#×&Ñ&Ø$Ð1ˆJà'Ñ3‘8¼ˆØÑØ!×'Ñ'¨Ñ+¨j¸©mÑ;ˆHØ"×(Ñ(¨Ñ,¨z¸!©}Ñ<ˆIØ#Ð/ˆKáØ#Ø'Ø!Ø1Ø+Ø%ØØ/Ø)Ø!Øñ
ð 	
r#   Úflattenc                 ó¤  • U(       a=  U R                   S   U R                   S   4nU R                  S   U R                  S   4nOU R                   4nU R                  4n/ UQU R                  PU R                  PU R                  PU R
                  PU R                  PU R                  PU R                  PU R                  PUQU R                  P7$ )z”
Returns a tuple of the attributes of the BlockMask.

Args:
    flatten (bool): If True, it will flatten the tuple of (KV_BLOCK_SIZE, Q_BLOCK_SIZE)
r   rZ   )rÿ   rø   r§   r¨   rù   rú   rû   rü   rý   rþ   r   )rÜ   r  Ú
block_sizerø   s       r!   Úas_tupleÚBlockMask.as_tuple®  s  € ö ØŸ/™/¨!Ñ,¨d¯o©o¸aÑ.@ÐAˆJØ×+Ñ+¨AÑ.°×0@Ñ0@ÀÑ0CÐD‰KàŸ/™/Ð+ˆJØ×+Ñ+Ð-ˆKð
Øð
à×Ñð
ð �O‰Oð
ð ×#Ñ#ð	
ð
 × Ñ ð
ð ×Ñð
ð �N‰Nð
ð ×"Ñ"ð
ð ×Ñð
ð ð
ð �M‰Mñ
ð 	
r#   c                 óf   • U R                   R                  Gt n  n[        U5      U R                  -   $ r�   )r¨   r²   rÕ   rø   )rÜ   r´   r¶   s      r!   r²   ÚBlockMask.shapeË  s.   € à ŸO™O×1Ñ1Ñˆ�Q˜Ü�ZÓ  4×#3Ñ#3Ñ3Ð3r#   c                 ó˜   • SU R                    SU R                  5       S S3nU R                  5       R                  5       nX-  nUS-  nU$ )NzBlockMask(shape=z, sparsity=ú.2fz%, 
ú
))r²   ÚsparsityÚ	to_stringÚstrip)rÜ   ÚsÚmask_strs      r!   Ú__str__ÚBlockMask.__str__Ð  sM   € Ø˜tŸz™z˜l¨+°d·m±m³oÀcÐ5JÈ%ÐPˆØ—>‘>Ó#×)Ñ)Ó+ˆØ	‰ˆØ	ˆU‰
ˆØˆr#   c                 ó6  • [        U[        5      (       d  U4OUn/ UQ[        S5      P[        S5      P[        S5      P7SS nU R                  R                  SS n[        S [        X#SS9 5       5      nU R                  U   nU R                  U   nU R                  b7  U R                  c  [        S5      eU R                  U   nU R                  U   nOSnSn[        R                  UUUUU R                  [        U R                  U R                  SLS9$ )am  
Returns a new BlockMask instance by getting the mask for the given index position.

Args:
    index: Index to apply to all attributes.

Example Usage:
    .. code-block:: python

        def causal_mask(b, h, q_idx, kv_idx):
            return q_idx >= kv_idx


        block_mask = create_block_mask(
            causal_mask, 4, 2, 512, 512, device="cuda"
        )
        assert block_mask.kv_num_blocks.shape == (4, 2, 4)
        assert block_mask.kv_indices.shape == (4, 2, 4, 4)

        # Index on batch dimension
        new_block_mask = block_mask[0]
        assert new_block_mask.kv_num_blocks.shape == (2, 4)
        assert new_block_mask.kv_indices.shape == (2, 4, 4)

        # Index on batch and head dimension
        new_block_mask = block_mask[0, 1]
        assert new_block_mask.kv_num_blocks.shape == (4,)
        assert new_block_mask.kv_indices.shape == (4, 4)

        # slicing on batch and head dimension
        new_block_mask = block_mask[0:2, 1:2]
        assert new_block_mask.kv_num_blocks.shape == (2, 1, 4)
        assert new_block_mask.kv_indices.shape == (2, 1, 4, 4)

        # slicing on batch, head, and query dimension
        new_block_mask = block_mask[
            0:2, 1:2, torch.tensor([1], dtype=torch.int32)
        ]
        assert new_block_mask.kv_num_blocks.shape == (2, 1, 1)
        assert new_block_mask.kv_indices.shape == (2, 1, 1, 4)
Nr[   c              3   ó¸   #   • U  HP  u  p[        U[        5      (       a2  U* Us=::  a  S :  a  O  O[        X-   X-   S-   5      O[        XS-   5      OUv •  MR     g7f)r   rZ   N)ræ   rL   Úslice)rh   ÚiÚns      r!   rj   Ú(BlockMask.__getitem__.<locals>.<genexpr>  sY   é € ð 
ò 8‘�ô ˜!œS×!Ñ!ð *+¨¨a­°!®ŒU�1‘5˜!™% !™)Ô$¼¸qÀaÁ%¼àôò 8ùs   ‚AAT)Ústrictr  )rÿ   r   rø   r  )ræ   rÕ   r!  r§   r²   Úzipr¨   rù   rú   ru   r$   r  rÿ   r—   rø   rü   )rÜ   ÚindexÚpaddedÚsizesÚnew_kv_num_blocksÚnew_kv_indicesÚnew_full_kv_num_blocksÚnew_full_kv_indicess           r!   Ú__getitem__ÚBlockMask.__getitem__×  s4  € ôT !+¨5´%× 8Ñ 8�‘¸eˆØ@�5Ð@œ% ›+Ð@¤u¨T£{Ð@´E¸$³KÑ@ÀÀ!ÐDˆØ×"Ñ"×(Ñ(¨¨!Ð,ˆÜñ 
ô ˜F°$Ò7ó	
ó 
ˆð !×.Ñ.¨uÑ5ÐØŸ™¨Ñ/ˆØ×"Ñ"Ñ.Ø×#Ñ#Ñ+Ü$Ð%GÓHÐHØ%)×%<Ñ%<¸UÑ%CÐ"Ø"&×"6Ñ"6°uÑ"=Ñà%)Ð"Ø"&ÐÜ×'Ñ'ØØØ"ØØ—‘Ü+Ø×(Ñ(Ø!Ÿ^™^°4Ð7ð (ð 	
ð 		
r#   c                 óL  • S[         R                  S -  4S jnSU R                  R                   SU R                  R                   SU" U R
                  5       SU" U R                  5       SU" U R                  5       SU" U R                  5       S	U" U R                  5       S
U" U R                  5       SU R                   SU R                   SU R                  5       S S[        U R                  S5      (       a  U R                  R                  OU R                   S3$ )NÚxc                 ó$   • U b  U R                   $ S $ r�   ©r²   )r1  s    r!   Úshape_or_noneÚ)BlockMask.__repr__.<locals>.shape_or_none   s   € Ø™m�1—7‘7Ð5°Ð5r#   zBlockMask(
    kv_num_blocks=z,
    kv_indices=z,
    full_kv_num_blocks=z,
    full_kv_indices=z,
    q_num_blocks=z,
    q_indices=z,
    full_q_num_blocks=z,
    full_q_indices=z,
    BLOCK_SIZE=z,
    shape=z,
    sparsity=r  z%,
    mask_mod=rG   r  )r   r   r§   r²   r¨   rù   rú   rû   rü   rý   rþ   rÿ   r  rn   r   rG   )rÜ   r4  s     r!   rñ   ÚBlockMask.__repr__  s8  € ð	6œUŸ\™\¨DÑ0ô 	6ð!Ø!%×!3Ñ!3×!9Ñ!9Ð :ð ;Ø"Ÿo™o×3Ñ3Ð4ð 5&Ù&3°D×4KÑ4KÓ&LÐ%Mð N#Ù#0°×1EÑ1EÓ#FÐ"Gð H Ù -¨d×.?Ñ.?Ó @ÐAð BÙ*¨4¯>©>Ó:Ð;ð <%Ù%2°4×3IÑ3IÓ%JÐ$Kð L"Ù"/°×0CÑ0CÓ"DÐ!Eð FØ"Ÿo™oÐ.ð /ØŸ™˜ð %Ø ŸM™M›O¨CÐ0ð 1Ü6=¸d¿m¹mÈZ×6XÑ6X˜DŸM™M×2Ò2Ð^b×^kÑ^kÐlð mðð	
r#   Ú	new_q_lenÚ
new_kv_lenc                 óÊ  • XR                   S   -   S-
  U R                   S   -  nX R                   S   -   S-
  U R                   S   -  n[        U R                  U R                  X45      u  pVU R                  b>  U R
                  c  [        S5      e[        U R                  U R
                  UU5      u  nnOS nS nU R                  UUUUU R                   U R                  5      $ )Nr   rZ   r  )	rÿ   rÌ   r§   r¨   rù   rú   ru   r  r   )	rÜ   r7  r8  rÉ   rÊ   r*  r+  r,  r-  s	            r!   Ú_adjustÚBlockMask._adjust4  sø   € Ø!§O¡O°AÑ$6Ñ6¸Ñ:¸t¿¹ÈqÑ?QÑQˆØ"§_¡_°QÑ%7Ñ7¸!Ñ;ÀÇÁÐPQÑ@RÑRˆÜ,JØ×Ñ §¡°ó-
Ñ)Ðð ×"Ñ"Ñ.Ø×#Ñ#Ñ+Ü$Ð%GÓHÐHô /Ø×'Ñ'Ø×$Ñ$ØØó	ñØ&Ù#ð &*Ð"Ø"&ÐØ×"Ñ"ØØØ"ØØ�O‰OØ�M‰Mó
ð 	
r#   c                 ó0   • U R                   nS nU" U5      $ )zIReturns the number of elements (not accounting for sparsity) in the mask.c                 óN   • [         R                  " [        R                  U S5      $ ©NrZ   )Ú	functoolsÚreduceÚoperatorÚmul)Úxss    r!   Ú_prodÚBlockMask.numel.<locals>._prodV  s   € Ü×#Ò#¤H§L¡L°"°aÓ8Ð8r#   r3  )rÜ   r²   rD  s      r!   ÚnumelÚBlockMask.numelR  s   € à—
‘
ˆò	9ñ �U‹|Ðr#   c                 ó   • U R                  5       nU R                  R                  5       nU R                  b  X R                  R                  5       -  nUR	                  5       U R
                  S   -  U R
                  S   -  nX1-  nSSU-
  -  $ )zEComputes the percentage of blocks that are sparse (i.e. not computed)r   rZ   éd   )rF  r§   rq   rù   Úitemrÿ   )rÜ   Ú
total_sizeÚcomputed_blocksÚcomputed_sizeÚdense_ratios        r!   r  ÚBlockMask.sparsity[  s†   € à—Z‘Z“\ˆ
Ø×,Ñ,×0Ñ0Ó2ˆØ×"Ñ"Ñ.Ø×6Ñ6×:Ñ:Ó<Ñ<ˆOà'×,Ñ,Ó.°·±ÀÑ1CÑCÀdÇoÁoÐVWÑFXÑXˆØ#Ñ0ˆØ�a˜+‘oÑ&Ð&r#   c                 óÖ   • [        U R                  U R                  5      nU R                  b;  U R                  c  [        S5      eU[        U R                  U R                  5      -  $ U$ )z;Returns a dense block that is equivalent to the block mask.r  )r¸   r§   r¨   rù   rú   ru   )rÜ   Úpartial_denses     r!   Úto_denseÚBlockMask.to_densef  sj   € ä)¨$×*<Ñ*<¸d¿o¹oÓNˆØ×"Ñ"Ñ.Ø×#Ñ#Ñ+Ü$Ð%GÓHÐHà Ô#4Ø×'Ñ'¨×)=Ñ)=ó$ñ ð ð Ðr#   c           
      ó  ^
^^^^• U R                  5       m
T
R                  Gt nmm[        U[        5      (       a  UmUmOUS:X  a  TmTmOUu  mmU
UUUU4S jn/ n[	        [
        R                  " U Vs/ s H  n[        U5      PM     sn6 5       HU  u  pxXr:X  a5  UR                  S5        UR                  S5        UR                  S5          OU" U6 n	UR                  U	5        MW     SR                  U5      $ s  snf )z—Returns a string representation of the block mask. Quite nifty.

If grid_size is -1, prints out an uncompressed version. Warning, it can be quite big!
r�   c            	      ó€  >• / nUR                  U  5        SR                  [        U5      5      S-   nS[        4S jnS n[	        SU" TT5      5      n[	        SU" TT5      5      n[        STU5       HF  n[        STU5       H-  nTn	U  H  n
Xš   n	M	     U" X—Xu-   2XˆU-   24   5      nX+S-  -  nM/     US-  nMH     U$ )	Nz, Ú
r   c                 óx   • U R                  5       R                  5       R                  5       nUS:X  a  gUS:X  a  gg)NrZ   u   â–ˆr   Ú u   â–‘)ÚfloatÚmeanrJ  )ÚsectionÚ
percentages     r!   Úsummarize_sectionÚHBlockMask.to_string.<locals>.create_block_vis.<locals>.summarize_section‰  s6   € Ø$Ÿ]™]›_×1Ñ1Ó3×8Ñ8Ó:�
Ø “?Ø Ø 1“_Øà r#   c                 ó   • XS-
  -   U-  $ r>  rF   )ÚaÚbs     r!   ÚcdivÚ;BlockMask.to_string.<locals>.create_block_vis.<locals>.cdiv’  s   € Ø ™U™¨Ñ)Ð)r#   rZ   r   r   )ÚappendÚjoinÚreversedrö   Úmaxr³   )Ú	batch_idxÚdescriptorsÚvisr]  rb  Úrow_stepÚcol_stepÚrÚcÚcur_maskÚidxÚcharr©   Úmax_colsÚmax_rowsr®   r¯   s               €€€€€r!   Úcreate_block_visÚ-BlockMask.to_string.<locals>.create_block_vis‚  sç   ø€ ØˆKà×Ñ ) Ô.à—)‘)œH [Ó1Ó2°TÑ9ˆCð!¬cô !ò*ô ˜1™d 8¨XÓ6Ó7ˆHÜ˜1™d 8¨XÓ6Ó7ˆHä˜1˜h¨Ö1�Ü˜q (¨HÖ5�AØ)�HÛ(˜Ø#+¡=šñ  )á,Ø  Q¡\Ð!1°1¸8±|Ð3CÐ!CÑDó�Dð  !™8‘O’Cñ 6ð �t‘’ñ 2ð ˆJr#   z...z3To print out more, set BlockMask.to_string(limit=N)zNYou can also index (BlockMask[batch, head]) to choose a specific batch or headrV  )
rR  r²   ræ   rL   Ú	enumerateÚ	itertoolsÚproductr³   rd  re  )rÜ   Ú	grid_sizeÚlimitr´   rt  Ú	total_visr"  rp  rh  Ú	block_visr©   rr  rs  r®   r¯   s             @@@@@r!   r  ÚBlockMask.to_stringr  s   ü€ ð
 —]‘]“_ˆ
Ø*4×*:Ñ*:Ñ'ˆ�X˜xÜ�i¤×%Ñ%Ø ˆHØ ‰HØ˜"‹_ØˆHØ‰Hà!*ÑˆH�h÷ 	ñ  	ðD ˆ	Ü'Ü×Ò±*Ó=²*¨Q¤ a¦±*Ñ=Ð>ö
‰NˆCð ‹|Ø× Ñ  Ô'Ø× Ñ Ð!VÔWØ× Ñ Ødôñ Ù(¨)Ð4ˆIØ×Ñ˜YÖ'ñ
ð �y‰y˜Ó#Ð#ùò  >s   Á5C<r’   c                 ól   ^• [        [        R                  U4S jU R                  SS95      n[	        U6 $ )a>  Moves the BlockMask to the specified device.

Args:
    device (torch.device or str): The target device to move the BlockMask to.
        Can be a torch.device object or a string (e.g., 'cpu', 'cuda:0').

Returns:
    BlockMask: A new BlockMask instance with all tensor components moved
    to the specified device.

Note:
    This method does not modify the original BlockMask in-place.
    Instead, it returns a new BlockMask instance where individual tensor attributes
    may or may not be moved to the specified device, depending on their
    current device placement.
c                 ó&   >• U R                  T5      $ r�   )r¿   )r1  r’   s    €r!   Ú<lambda>ÚBlockMask.to.<locals>.<lambda>Ç  s   ø€ �a—d‘d˜6”lr#   F)r  )r   r   r   r  r$   )rÜ   r’   Úmapped_attributess    ` r!   r¿   ÚBlockMask.to´  s6   ø€ ô" *Ü�L‰LÜ"Ø�M‰M %ˆMÐ(ó
Ðô
 Ð+Ð,Ð,r#   ÚattrÚvaluec                 ó(   • U S:X  a  [        U5      $ U$ )Nr   )rÙ   ©r„  r…  s     r!   Ú_wrap_context_valueÚBlockMask._wrap_context_valueÌ  s   € à�:ÓÜ" 5Ó)Ð)Øˆr#   c                 ó‚   • U S:X  a8  [        U[        5      (       d  [        S[        U5       35      eUR                  $ U$ )Nr   zExpected _MaskModWrapper, got )ræ   rÙ   ru   Útyper_   r‡  s     r!   Ú_unwrap_context_valueÚBlockMask._unwrap_context_valueÒ  s<   € à�:ÓÜ˜e¤_×5Ñ5Ü$Ð'EÄdÈ5ÃkÀ]Ð%SÓTÐTØ—8‘8ˆOØˆr#   c                 ó†   ^ • [        U 4S jT R                   5       5      n[        U 4S jT R                   5       5      nX4$ )z�Flatten BlockMask into a list of tensors and context.

Wraps mask_mod in _MaskModWrapper for value-based comparison in TreeSpec.
c              3   ó<   >#   • U  H  n[        TU5      v •  M     g 7fr�   )rÔ   ©rh   r„  rÜ   s     €r!   rj   Ú%BlockMask._flatten.<locals>.<genexpr>ß  s   øé € ÐKÒ8J°œ  d×+Ð+Ò8Jùs   ƒc              3   ó\   >#   • U  H!  nTR                  U[        TU5      5      v •  M#     g 7fr�   )rˆ  rÔ   r�  s     €r!   rj   r‘  à  s/   øé € ð 
â+�ð ×$Ñ$ T¬7°4¸Ó+>×?Ð?Ú+ùs   ƒ),©rÕ   Ú_TENSOR_ATTRSÚ_CONTEXT_ATTRS©rÜ   ÚtensorsÚcontexts   `  r!   Ú_flattenÚBlockMask._flattenÚ  sC   ø€ ô
 ÔK¸×8JÒ8JÓKÓKˆÜô 
à×+Ò+ó
ó 
ˆð ÐÐr#   c           	      óÚ   • [        U R                  U5       VVs0 s H  u  p4X0R                  X45      _M     nnnUR                  [        U R                  U5      5        U " S0 UD6$ s  snnf )z4Unflatten tensors and context back into a BlockMask.rF   )r&  r•  rŒ  Úupdater”  )r	  r—  r˜  r„  Úvalrá   s         r!   Ú
_unflattenÚBlockMask._unflattenæ  sm   € ô
 ! ×!3Ñ!3°WÔ=ô
â=‘	�ð ×+Ñ+¨DÓ6Ò6Ù=ð 	ñ 
ð 	�‰”c˜#×+Ñ+¨WÓ5Ô6á‰}�V‰}Ðùó
s   šA'c                 ó†   ^ • [        U 4S jT R                   5       5      n[        U 4S jT R                   5       5      nX4$ )z{Flatten BlockMask with keys for better tracing.

Wraps mask_mod in _MaskModWrapper for value-based comparison in TreeSpec.
c              3   óR   >#   • U  H  n[        U5      [        TU5      4v •  M     g 7fr�   )r   rÔ   r�  s     €r!   rj   Ú/BlockMask._flatten_with_keys.<locals>.<genexpr>ö  s&   øé € ð 
Ú@R¸ŒZ˜Óœw t¨TÓ2Õ3Ò@Rùs   ƒ$'c           	   3   ór   >#   • U  H,  n[        U5      TR                  U[        TU5      5      4v •  M.     g 7fr�   )r   rˆ  rÔ   r�  s     €r!   rj   r¢  ù  s6   øé € ð 
â+�ô ˜Ó˜t×7Ñ7¸¼gÀdÈDÓ>QÓRÕSÚ+ùs   ƒ47r“  r–  s   `  r!   Ú_flatten_with_keysÚBlockMask._flatten_with_keysñ  sL   ø€ ô
 ô 
Ø@D×@RÒ@Ró
ó 
ˆô ô 
à×+Ò+ó
ó 
ˆð ÐÐr#   )rÿ   rú   rù   rþ   rý   r¨   r§   r   rü   rû   rø   )T)r   r$   ))é   r¦  rm   ))rG   rH   rI   rJ   rK   rÕ   rL   rM   r   rô   r”  r•  rÝ   ÚclassmethodÚ_DEFAULT_SPARSE_BLOCK_SIZErN   r  r  Úpropertyr²   rö   r  r.  rñ   r:  rF  rY  r  rR  r  r   r’   r¿   Ústaticmethodr   rˆ  rŒ  r™  rž  r¤  rO   rF   r#   r!   r$   r$   Þ  sÐ  ‡ ñ:ðx �s˜C�x‘Ó ØÓØÓØ ™Ó%Ø˜d‘]Ó"Ø˜4‘-ÓØ˜‰}ÓØ ‘}Ó$Ø˜T‘MÓ!Ø�c˜3�h‘ÓØ!Ó!ò	€Mò€Nð'!à˜3 ˜8‘_ð'!ð ð'!ð ð	'!ð
 # T™Mð'!ð   $™ð'!ð ˜t‘mð'!ð ˜D‘=ð'!ð " D™=ð'!ð  ™ð'!ð ˜#˜s˜(‘Oð'!ð &ð'!ð 
ô'!ðR ð
 -1Ø)-Ø,FØ/3Ø.2Ø!%ñI
àðI
ð ðI
ð # T™Mð	I
ð
   $™ðI
ð ˜%  S ™/Ñ)ðI
ð &¨Ñ,ðI
ð ˜3 ˜8‘_ tÑ+ðI
ð ôI
ó ðI
ñV
 õ 
ð: ñ4ó ð4ð˜ô ôF
ðP
˜#ô 
ð*
 ð 
°#ô 
ò<ð	'˜%ô 	'ð
˜&ô 
ô@$ðD-˜Ÿ™¨Ñ+ð -°ô -ð0 ð #ð ¨cð °có ó ðð
 ð Cð °ð ¸ó ó ðò
 ð ñó ðõ r#   r$   c                 ó|   • U R                  5       U:  a'  U R                  S5      n U R                  5       U:  a  M'  U $ )Nr   )r»   r¤   )r1  r»   s     r!   Ú_broadcast_to_dimr¬     s0   € Ø
�%‰%‹'�C‹-Ø�K‰K˜‹Nˆð �%‰%‹'�C�-à€Hr#   c                 ó   • X-   S-
  U-  U-  $ r>  rF   ©r1  Úmultiples     r!   Ú_round_up_to_multipler°    s   € Ø‰L˜1Ñ Ñ)¨HÑ4Ð4r#   ÚmaskÚseparate_full_blocksc           
      ó6  • U R                   [        R                  :w  a  [        SU R                    35      e[	        U S5      n S n[        R
                  R                  R                  U SU" U R                  S   U5      SU" U R                  S   U5      45      n U R                  u  pVpxXq-  S:w  a  [        SU SU S	35      eX‚-  S:w  a  [        S
U SU S	35      eU R                  XVXq-  XU-  U5      n U R                  SSSSSS5      n U R                  SS/S9n	U(       aO  X-  n
Xš:H  nU	S:„  Xš:  -  nUR                  [        R                  S9nUR                  [        R                  S9nXË4$ U	S:„  nUR                  [        R                  S9nUS 4$ )Nz#mask.dtype must be torch.bool, got rm   c                 ó   • [        X5      U -
  $ r�   )r°  r®  s     r!   Úpadding_needed_for_multipleÚ@_convert_mask_to_block_mask.<locals>.padding_needed_for_multiple  s   € Ü$ QÓ1°AÑ5Ð5r#   r   r�   rœ   zQ (z%) must be divisible by Q_BLOCK_SIZE (rð   zKV (z&) must be divisible by KV_BLOCK_SIZE (rZ   r   r[   rl   rº   rŸ   )r‘   r   rN   ru   r¬  ÚnnÚ
functionalÚpadr²   ÚviewÚpermuterq   r¿   Úint8)r±  ÚQ_BLOCK_SIZEÚKV_BLOCK_SIZEr²  rµ  ÚBÚHÚQÚKVÚmask_block_sumÚfull_block_sumÚfull_blocksÚpartial_blockss                r!   Ú_convert_mask_to_block_maskrÇ  
  sÒ  € ð ‡z�z”U—Z‘ZÓÜÐBÀ4Ç:Á:À,ÐOÓPÐPÜ˜T 1Ó%€Dò6ô �8‰8×Ñ×"Ñ"ØàÙ'¨¯
©
°2©¸ÓFØÙ'¨¯
©
°2©¸ÓEð		
ó€Dð —*‘*�K€Aˆ!ØÑ˜1ÓÜØ�!�Ð9¸,¸ÀqÐIó
ð 	
ð 
Ñ˜QÓÜØ�2�$Ð<¸]¸OÈ1ÐMó
ð 	
ð �9‰9Ø	ˆaÑ °]Ñ/BÀMó€Dð �<‰<Ø	ˆ1ˆa��A�qó€Dð —X‘XØ�ˆHð ð €Nö Ø%Ñ5ˆØ$Ñ6ˆØ(¨1Ñ,°Ñ1PÑQˆØ'×*Ñ*´·±Ð*Ð<ˆØ!—n‘n¬5¯:©:�nÐ6ˆØÐ*Ð*à'¨!Ñ+ˆØ'×*Ñ*´·±Ð*Ð<ˆØ˜tÐ#Ð#r#   Ú	mask_modsc                  ó^   ^ • [        S T  5       5      (       d  [        ST  35      eU 4S jnU$ )z9Returns a mask_mod that's the union of provided mask_modsc              3   ó8   #   • U  H  n[        U5      v •  M     g 7fr�   ©Úcallable©rh   Úargs     r!   rj   Úor_masks.<locals>.<genexpr>A  ó   é € Ð2ª	 Œx˜�}ˆ}ª	ùrÓ   ú)All inputs should be callable mask_mods: c                 ól   >• U R                  S[        R                  S9nT H  nXE" XX#5      -  nM     U$ ©NrF   rŸ   )r¡   r   rN   ©ra  ÚhÚq_idxÚkv_idxÚresultr±  rÈ  s         €r!   Úor_maskÚor_masks.<locals>.or_maskD  s9   ø€ Ø—‘˜R¤u§z¡z�Ð2ˆÛˆDØ˜d 1¨Ó7Ñ7ŠFñ àˆr#   ©Úallr–   )rÈ  rÙ  s   ` r!   r*   r*   ?  s3   ø€ äÑ2©	Ó2×2Ñ2ÜÐFÀyÀkÐRÓSÐSõð €Nr#   c                  ó^   ^ • [        S T  5       5      (       d  [        ST  35      eU 4S jnU$ )z@Returns a mask_mod that's the intersection of provided mask_modsc              3   ó8   #   • U  H  n[        U5      v •  M     g 7fr�   rË  rÍ  s     r!   rj   Úand_masks.<locals>.<genexpr>O  rÐ  rÓ   rÑ  c                 ól   >• U R                  S[        R                  S9nT H  nXE" XX#5      -  nM     U$ rÓ  )r“   r   rN   rÔ  s         €r!   Úand_maskÚand_masks.<locals>.and_maskR  s9   ø€ Ø—‘˜B¤e§j¡j�Ð1ˆÛˆDØ˜d 1¨Ó7Ñ7ŠFñ àˆr#   rÛ  )rÈ  rá  s   ` r!   r+   r+   M  s3   ø€ äÑ2©	Ó2×2Ñ2ÜÐFÀyÀkÐRÓSÐSõð €Or#   c                 ó  • U R                  5       S:w  a  [        SU R                  5        35      eU R                  u  p4pVU R                  " X!/U R                  Q76 n U R	                  SSSSSS5      R                  X4XR-  Xa-  5      n U $ )Nrm   z block_mask.dim() must be 4, got r   r[   r   rl   rZ   )r»   ru   r²   Úexpandr»  Úreshape)Ú
block_maskr¾  r½  r¿  rÀ  rÁ  rÂ  s          r!   Ú_convert_block_mask_to_maskrç  [  s’   € ð
 ‡~�~Ó˜1ÓÜÐ?À
ÇÁÓ@PÐ?QÐRÓSÐSØ×"Ñ"�K€Aˆ!Ø×"Ò" <ÐRÀ×AQÑAQÒR€JØ×#Ñ# A q¨!¨Q°°1Ó5×=Ñ=Ø	ˆaÑ Ñ 2ó€Jð Ðr#   ræ  r   rø   r½  r¾  c           
      óŽ   • U u  pV[        U5      nUb  [        U5      nOSn[        R                  US   US   US   US   X44UUS9$ )Nr  r   rZ   )rÿ   r   rø   )rÂ   r$   r  )	ræ  r   rø   r½  r¾  rÆ  rÅ  Ú
partial_bmÚfull_bms	            r!   Ú$_create_sparse_block_from_block_maskrë  j  sk   € ð #-Ñ€Nä" >Ó2€JØÑÜ7HÈÓ7U‰àˆä×#Ñ#Ø�1‰Ø�1‰Ø�‰
Ø�‰
Ø Ð0ØØð $ð ð r#   Úmod_fnr¿  rÀ  ÚQ_LENÚKV_LENr’   c                 óÄ  • Uc'  [         R                  R                  5       =(       d    SnUc  SnUc  Sn[         R                  " SXS9n[         R                  " SX%S9n[         R                  " SX5S9n[         R                  " SXES9n	[	        U 5      n
SSKJn  U" 5          U
[        R                  :X  aa  U n[        USS9nU" [         R                  " XX4US9XgX‰5      n[         R                  " [         R                  " U5      S	S
5      nUsSSS5        $ U
[        R                  :X  a   U n[        USS9nU" XgX‰5      nUsSSS5        $ [        e! , (       d  f       g= f)a¹  This function creates a mask tensor from a mod_fn function.

Args:
    mod_fn (Union[_score_mod_signature, _mask_mod_signature]): Function to modify attention scores.
    B (int): Batch size.
    H (int): Number of query heads.
    Q_LEN (int): Sequence length of query.
    KV_LEN (int): Sequence length of key/value.
    device (str): Device to run the mask creation on.

Returns:
    mask (Tensor): A mask tensor with shape (B, H, M, N).
NÚcpurZ   r   )r’   )ÚTransformGetItemToIndex)r   )r|   FTrF   )r   ÚacceleratorÚcurrent_acceleratorr£   r{   Ú,torch._dynamo._trace_wrapped_higher_order_oprñ  rX   r\   r†   Úzerosr¥   Úisneginfr]   ru   )rì  r¿  rÀ  rí  rî  r’   ra  rÕ  Úmr#  Úmod_typerñ  Ú	score_modr·   r±  r   s                   r!   r)   r)   „  s5  € ð* �~Ü×"Ñ"×6Ñ6Ó8×A¸EˆØ�yØˆØ�yØˆÜ�Š�Q˜Ñ)€AÜ�Š�Q˜Ñ)€AÜ�Š�Q˜Ñ-€AÜ�Š�Q˜Ñ.€AÜ˜VÓ$€HåTá	 Õ	"ØÔ(×2Ñ2Ó2ØˆIÜ'¨	¸$Ñ?ˆIÙœEŸKšK¨¨eÀFÑKÈQÐSTÓXˆCÜ—;’;œuŸ~š~¨cÓ2°E¸4Ó@ˆDØ÷ 
#Ñ	"ð Ô*×3Ñ3Ó3ØˆHÜ& x¸Ñ;ˆHÙ˜A !Ó'ˆDØ÷ 
#Ñ	"ô !Ð ÷ 
#Õ	"ús   Â!A,EÄ*EÅEÅ
Erÿ   c           	      óä  • Uc'  [         R                  R                  5       =(       d    Sn[        U 5      nU[        R
                  :w  a  [        SU  35      eUc  SnUc  Sn[        U[        5      (       a  Un	Un
OUu  pšU(       a=  [        R                  " S[        SS9  [         R                  " [        5      " XX#XEU5      $ [        XX#XE5      n[        UU	U
SS9u  pÍ[!        XÍ4U X44U	U
5      nU$ )	a‰  This function creates a block mask tuple from a mask_mod function.

Args:
    mask_mod (Callable): mask_mod function. This is a callable that defines the
        masking pattern for the attention mechanism. It takes four arguments:
        b (batch size), h (number of heads), q_idx (query index), and kv_idx (key/value index).
        It should return a boolean tensor indicating which attention connections are allowed (True)
        or masked out (False).
    B (int): Batch size.
    H (int): Number of query heads.
    Q_LEN (int): Sequence length of query.
    KV_LEN (int): Sequence length of key/value.
    device (str): Device to run the mask creation on.
    BLOCK_SIZE (int or tuple[int, int]): Block size for the block mask. If a single int is provided it is used for both query and key/value.

Returns:
    BlockMask:  A BlockMask object that contains the block mask information.

Example Usage:
    .. code-block:: python

        def causal_mask(b, h, q_idx, kv_idx):
            return q_idx >= kv_idx


        block_mask = create_block_mask(causal_mask, 1, 1, 8192, 8192, device="cuda")
        query = torch.randn(1, 1, 8192, 64, device="cuda", dtype=torch.float16)
        key = torch.randn(1, 1, 8192, 64, device="cuda", dtype=torch.float16)
        value = torch.randn(1, 1, 8192, 64, device="cuda", dtype=torch.float16)
        output = flex_attention(query, key, value, block_mask=block_mask)
rð  z4create-block_mask requires a mask_mod function! Got rZ   a  _compile flag on create_block_mask was originally added to work around a torch.compile limitation. That limitation has since been addressed. So, to compile create_block_mask, we suggest doing torch.compile(create_block_mask). This still works for now, but will be removed in the future.r   r   T)r½  r¾  r²  )r   rò  ró  r{   rX   r]   ru   ræ   rL   r   r   ÚDeprecationWarningÚcompiler(   r)   rÇ  rë  )r   r¿  rÀ  rí  rî  r’   rÿ   Ú_compilerø  r½  r¾  Úmask_tensorÚpartial_block_maskÚfull_block_maskræ  s                  r!   r(   r(   ·  s  € ðR �~Ü×"Ñ"×6Ñ6Ó8×A¸EˆÜ˜XÓ&€HØÔ$×-Ñ-Ó-ÜØBÀ8À*ÐMó
ð 	
ð 	�yØˆØ�yØˆÜ�*œc×"Ñ"Ø!ˆØ"‰à&0Ñ#ˆæÜ�Šð mÜØò	
ô
 �}Š}Ô.Ô/Ø˜ 6°:ó
ð 	
ô ˜h¨1°VÓD€KÜ*EØØ!Ø#Ø!ñ	+Ñ'Ðô 6Ø	Ð-ØØ	ˆØØó€Jð Ðr#   ÚqueryÚkeyc           	      óÞ   • U R                   n[        R                  [        R                  " / SQ[        R
                  US9[        R                  " / SQ[        R
                  US9[        SS9$ )zßDefault block mask for flex attention.
If users don't specify any block sparse mask info, we create this
empty block sparse mask. Which creates a BlockMask with 1 block that is the full length
of the query and key tensors.
)rZ   rZ   rZ   r    )rZ   rZ   rZ   rZ   )rZ   rZ   )r§   r¨   rÿ   rø   )r’   r$   r  r   Úonesr¢   rõ  Ú_LARGE_SPARSE_BLOCK_SIZE)r  r  r’   s      r!   Ú_create_empty_block_maskr    sS   € ð �\‰\€FÜ×#Ñ#Ü—j’j¢´%·+±+ÀfÑMÜ—;’;š|´5·;±;ÀvÑNÜ+Øð	 $ð ð r#   r…  Ú
return_lseÚ
return_auxc                 ór  • Uc  0 O
[        U5      nSU;   a"  UR                  SS5      (       a  [        S5      eSU;   a7  [        R                  " [
        5      nUS   U;  a  [        SUS    SU 35      eUR                  SS5        UR                  SS5        UR                  S	S5        UR                  S
S5        UR                  SS5        U R                  R                  S:H  =(       d9    UR                  R                  S:H  =(       d    UR                  R                  S:H  nUnSn	Ub  UR                  nUR                  n	SU;   a  [        S5      eSUS'   U(       d$  [        R                  " 5       US'   U(       a  SUS'   SU;   a  [        S5      eX”S'   U(       a  U	(       a  [        S5      eU$ )NrE   r@   Fz¯BACKEND cannot be combined with legacy FORCE_USE_FLEX_ATTENTION. BACKEND supersedes the legacy knob; please drop FORCE_USE_FLEX_ATTENTION and only specify the desired BACKEND.zInvalid BACKEND value 'z'. Must be one of r-   r<   r=   r>   r?   Trð  ÚOUTPUT_LOGSUMEXPz.OUTPUT_LOGSUMEXP must not be in kernel_optionsÚ
OUTPUT_MAXz(OUTPUT_MAX must not be in kernel_optionsz-Returning max scores is not supported on CPU.)ÚdictÚgetr–   ÚtypingÚget_argsr1   Ú
ValueErrorÚ
setdefaultr’   r‹  rS   rT   ru   r   Úis_grad_enabledÚNotImplementedError)
r  r  r…  r  Úkernel_optionsr  Úvalid_backendsÚany_inputs_on_cpu_deviceÚ
output_lseÚ
output_maxs
             r!   Ú_apply_kernel_optionsr    sÚ  € ð *Ñ1‘R´t¸NÓ7K€Nà�NÓ" ~×'9Ñ'9Ø" E÷(ñ (ô ð4ó
ð 	
ð �NÓ"ÜŸš¬Ó2ˆØ˜)Ñ$¨NÓ:ÜØ)¨.¸Ñ*CÐ)Dð E"Ø"0Ð!1ð3óð ð
 ×Ñ˜i¨Ô0Ø×Ñ˜m¨UÔ3Ø×ÑÐ4°eÔ<Ø×ÑÐ5°uÔ=à×Ñ˜j¨$Ô/ð 	�‰×Ñ˜UÑ"÷ 	&Ø�:‰:�?‰?˜eÑ#÷	&à�<‰<×Ñ Ñ%ð ð €JØ€JàÑà—^‘^ˆ
Ø×*Ñ*ˆ
ð ˜^Ó+ÜÐMÓNÐNØ)-€NÐ%Ñ&Þô .3×-BÒ-BÓ-DˆÐ)Ñ*Þ#ð 27ˆNÐ-Ñ.ð �~Ó%ÜÐGÓHÐHØ#-�<Ñ Þ¦Jô "Ð"QÓRÐRð Ðr#   c                 ó¬   • U R                  S5      UR                  S5      :w  a0  [        SU R                  S5       SUR                  S5       S35      eg )Nr�   zJExpect query and key/value to have the same embedding dimension but got E=z and E=Ú.)r�   r  )r  r  r…  s      r!   Ú_validate_embed_dimr  g  sU   € Ø‡z�z�"ƒ~˜Ÿ™ "›Ó%ÜðØŸ™ B›Ð(¨°·±¸³¨~¸Qð@ó
ð 	
ð &r#   c                 ó6  • U R                   R                  S:X  a>  U R                  (       d"  UR                  (       d  UR                  (       a  [        S5      e1 SknU R                   R                  U;  a#  [	        SU R                   R                   S35      eg)zŒTODO: Remove once non cuda/cpu devices support is added
We only need to check query since we have already that q,k,v are on the same device
rð  zrFlexAttention does not support backward on CPU. Please set the input requires_grad to False or use another device.>   rð  ÚhpuÚxpuÚcudazTFlexAttention is only supported on CUDA, CPU or HPU devices. Found input tensors on z device.N)r’   r‹  Úrequires_gradr  r  )r  r  r…  Úsupported_devicess       r!   Ú_validate_devicer#  o  sŒ   € ð ‡|�|×Ñ˜EÓ!Ø××˜s×0×0°E×4G×4Gä!ð Aó
ð 	
ò 6ÐØ‡|�|×ÑÐ 1Ó1Üð&Ø&+§l¡l×&7Ñ&7Ð%8¸ðBó
ð 	
ð 2r#   c                 óœ  • S[         S[        4S jnS[         S[        4S jn[        R                  [        R                  4nU R
                  nXe;   =(       an    [        R                  R                  SL=(       aK    [        R                  R                  S5      S:¬  =(       a"    [        R                  R                  S5      S:  nU(       d  XU4$ U" U 5      (       d  U R                  5       n U" U5      (       d  UR                  5       nU" U5      (       d0  UR                  S	S
5      R                  5       R                  S	S
5      nXU4$ )a£  
Enforce memory layouts for query, key, and value tensors.

For non-FP8 dtypes, no action is taken.

For FP8 dtypes, we enforce the following memory layouts:
- Query tensor must be in row-major memory layout, as it will be the left-operand in the FP8 GEMM `q @ k.T`.
- Key tensor must be in row-major memory layout, as it will be transposed when used as the right-operand
  in the FP8 GEMM `q @ k.T`, meaning it will correctly be in column-major memory layout for the GEMM.
- Value tensor must be in column-major memory layout, as it will be the right-operand in the FP8 GEMM `softmax_scores @ v`.

Returns the query, key, and value tensors with the enforced memory layouts.
Útensorr   c                 ó.   • U R                  5       S   S:H  $ )Nr�   rZ   ©Ústride©r%  s    r!   Úis_row_majorÚ*_enforce_mem_layouts.<locals>.is_row_major’  ó   € Ø�}‰}‹˜rÑ" aÑ'Ð'r#   c                 ó.   • U R                  5       S   S:H  $ )Nrœ   rZ   r'  r)  s    r!   Úis_col_majorÚ*_enforce_mem_layouts.<locals>.is_col_major•  r,  r#   Nr   )é   é	   )é
   r   rœ   r�   )r   rN   r   Úfloat8_e4m3fnÚfloat8_e5m2r‘   Úversionr   Úget_device_capabilityr¦   rÄ   )r  r  r…  r*  r.  Ú
fp8_dtypesÚgemm_precisionÚshould_enforce_mem_layouts           r!   Ú_enforce_mem_layoutsr:  �  s.  € ð"(œVð (¬ô (ð(œVð (¬ô (ô 	×ÑÜ×Ñð€Jð —[‘[€Nð 	Ñ$÷ 	?Ü�M‰M×Ñ dÐ*÷	?ä�J‰J×,Ñ,¨VÓ4¸Ñ>÷	?ô �J‰J×,Ñ,¨VÓ4°wÑ>ð	 ö %Ø˜5Ð Ð ñ ˜×ÑØ× Ñ Ó"ˆñ ˜×ÑØ�n‰nÓˆñ ˜×ÑØ—‘  BÓ'×2Ñ2Ó4×>Ñ>¸rÀ2ÓFˆØ�uÐÐr#   )r  rù  ÚscaleÚ
enable_gqar  c	          
      óæ  • [        XUSS9  [        XU5        [        XU5        [        XU5      u  pnU R	                  5       S:w  d(  UR	                  5       S:w  d  UR	                  5       S:w  a  [        S5      eU(       dT  U R                  S5      UR                  S5      :w  a0  [        SU R                  S5       SUR                  S5       S35      eU(       a<  U R                  S	5      n
UR                  S	5      nX«-  S
:w  a  [        SU
 SU S35      eU R                  S
5      UR                  S
5      :w  a­  Uc0  [        SU R                  S
5       SUR                  S
5       S35      eUR                  R                  S
5      U R                  S
5      :w  aL  [        SU R                  S
5       SUR                  S
5       SUR                  R                  S
5       S35      eUc  [        nUc  [        X5      n[        USS5      [        L a  [        S5      eUR                  S
   [        :X  a  UR                  S	   [        :X  a  GO‚UR                   S   nUR                   S   nU R                  S5      U:”  d  UR                  S5      U:”  a=  [        SUR                    SU R                  S5       SUR                  S5       S35      eU R                  S5      U:  a  UR                  S5      U::  d*  U R                  S5      U::  aR  UR                  S5      U:  a=  [        SUR                    SU R                  S5       SUR                  S5       S35      eU R                  S5      U:w  a!  [#        SU R                  S5       SU S35      eUR                  S5      U:w  a!  [#        SUR                  S5       S U S35      eUc(  S![$        R&                  " U R                  S5      5      -  nU R(                  UR                  R(                  :w  a0  [        S"U R(                   S#UR                  R(                   S35      eU(       a  U	b  [        S$5      eU(       a  U	c  [+        S%S&[,        S'9  [/        U UUUUU	5      nS([0        S-  S)[2        4S* jn[4        R6                  R9                  5       (       aw  XU4 HC  n[4        R:                  R=                  US5        [4        R:                  R=                  US5        ME     [?        U UUUURA                  5       UU5      u  nnnU" UUUX—S+9$ [B        (       d
  [+        S,S-S.9  [4        R:                  RE                  5       (       d  [        S/5      eS0 n[G        5        n[B        (       a  UnO[4        RH                  " UUSS19nU" U UUUURA                  5       UU5      u  nnnSSS5        U" WWWX—S+9$ ! , (       d  f       N= f)2aÂ  This function implements scaled dot product attention with an arbitrary attention score modification function
described in the `Flex Attention <https://arxiv.org/abs/2412.05496>`_ paper. See also the
`blog post <https://pytorch.org/blog/flexattention/>`_.

This function computes the scaled dot product attention between query, key, and value tensors with a user-defined
attention score modification function. The attention score modification function will be applied after the attention
scores have been calculated between the query and key tensors. The attention scores are calculated as follows:

The ``score_mod`` function should have the following signature:

.. code-block:: python

    def score_mod(
        score: Tensor,
        batch: Tensor,
        head: Tensor,
        q_idx: Tensor,
        k_idx: Tensor
    ) -> Tensor:

Where:
    - ``score``: A scalar tensor representing the attention score,
      with the same data type and device as the query, key, and value tensors.
    - ``batch``, ``head``, ``q_idx``, ``k_idx``: Scalar tensors indicating
      the batch index, query head index, query index, and key/value index, respectively.
      These should have the ``torch.int`` data type and be located on the same device as the score tensor.

Args:
    query (Tensor): Query tensor; shape :math:`(B, Hq, L, E)`. For FP8 dtypes, should be in row-major memory layout for optimal performance.
    key (Tensor): Key tensor; shape :math:`(B, Hkv, S, E)`. For FP8 dtypes, should be in row-major memory layout for optimal performance.
    value (Tensor): Value tensor; shape :math:`(B, Hkv, S, Ev)`. For FP8 dtypes, should be in column-major memory layout for optimal performance.
    score_mod (Optional[Callable]): Function to modify attention scores. By default no score_mod is applied.
    block_mask (Optional[BlockMask]): BlockMask object that controls the blocksparsity pattern of the attention.
    scale (Optional[float]): Scaling factor applied prior to softmax. If none, the default value is set to :math:`\frac{1}{\sqrt{E}}`.
    enable_gqa (bool): If set to True, enables Grouped Query Attention (GQA) and broadcasts key/value heads to query heads.
    return_lse (bool): Whether to return the logsumexp of the attention scores. Default is False. **Deprecated**: Use ``return_aux=AuxRequest(lse=True)`` instead.
    kernel_options (Optional[FlexKernelOptions]):
        Options to control the behavior of the underlying Triton kernels.
        See :class:`FlexKernelOptions` for available options and usage examples.
    return_aux (Optional[AuxRequest]): Specifies which auxiliary outputs to compute and return.
        If None, only the attention output is returned. Use ``AuxRequest(lse=True, max_scores=True)``
        to request both auxiliary outputs.

Returns:
    output (Tensor): Attention output; shape :math:`(B, Hq, L, Ev)`.

    When ``return_aux`` is not None:
        aux (AuxOutput): Auxiliary outputs with requested fields populated.

    When ``return_aux`` is None (deprecated paths):
        lse (Tensor): Log-sum-exp of attention scores; shape :math:`(B, Hq, L)`. Only returned if ``return_lse=True``.

Shape legend:
    - :math:`N: \text{Batch size} ... : \text{Any number of other batch dimensions (optional)}`
    - :math:`S: \text{Source sequence length}`
    - :math:`L: \text{Target sequence length}`
    - :math:`E: \text{Embedding dimension of the query and key}`
    - :math:`Ev: \text{Embedding dimension of the value}`

.. warning::
    `torch.nn.attention.flex_attention` is a prototype feature in PyTorch.
    Please look forward to a more stable implementation in a future version of PyTorch.
    Read more about feature classification at: https://pytorch.org/blog/pytorch-feature-classification-changes/#prototype

T)Úallow_lowp_kvrm   z-NYI: query, key, and value must be 4D tensorséýÿÿÿzGExpect query and key/value to have the same number of heads but got Hq=z	 and Hkv=z&. Try setting enable_gqa=True for GQA.rZ   r   zMExpect number of query heads to be a multiple of kv heads for GQA but got Hq=r  NzlExpect query and key/value to have the same batch size, or non-none block_mask, but got block_mask=None, Bq=z
, and Bkv=zxExpect query and key/value to have the same batch size, or block_mask and query to have the same batch size, but got Bq=z, Bkv=z, B_block_mask=r   z+Cannot use mask_mod from a sliced BlockMaskrœ   r�   z,block_mask was created for block_mask.shape=z but got q_len=z and kv_len=zz. As the block mask was created for a smaller length than you're using it for, you likely need to create a new block mask.ad  . As the block mask was created for a larger length than you're using it for, you can either 1. create a new block mask with the correct length, or 2. 'adjust' the existing block mask to the correct length by calling block_mask._adjust(q_len, kv_len). This essentially 'crops' the block mask to the upper left corner, which does not work for all mask_mods!zquery.size(-2) (z) != block_mask_q_len (rð   zkey.size(-2) (z) != block_mask_kv_len (g      ð?z=Expect q/k/v and block_mask to be on the same device but got z and z|Cannot specify both return_lse and return_aux. return_lse is deprecated, please use return_aux=AuxRequest(lse=True) instead.Údeprecated_return_lsezjreturn_lse is deprecated and will be removed in v2.10. Please use return_aux=AuxRequest(lse=True) instead.)r   r  r  c                óV  • [         R                  " S5      nU=(       d    USL=(       a    UR                  nUSL=(       a    UR                  nU(       a  UR	                  5       S:”  a  X-  OSnU(       a  UR	                  5       S:”  a  X%-  OSnUb  U [        UUS94$ U(       a  X4$ U $ )zFNormalize stats and build return value (aux-aware, legacy-compatible).g       @Nr   )rS   rT   )ÚmathÚlogrS   rT   rF  r%   )	r·   rS   rT   r  r  Úln2Ú
return_maxÚ
lse_scaledÚ
max_scaleds	            r!   Ú_finalize_outputsÚ)flex_attention.<locals>._finalize_outputsw  s¨   € ô �hŠh�s‹mˆØ×L :°TÐ#9×#L¸j¿n¹nˆ
Ø tÐ+×E°
×0EÑ0Eˆ
æ#-°#·)±)³+À³/�S’YÈˆ
æ!+°
×0@Ñ0@Ó0BÀQÓ0FˆJÒÈTð 	ð Ñ!Øœ	ØØ%ñð ð ö
 Ø�?Ð"àˆ
r#   )r  r  Úflex_attention_performanceaó  flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.

SOLUTION: Use torch.compile(flex_attention)(...)

If you want to debug your score_mod/mask_mod, you can set:
torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True

This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.)r   r   z&flex_attention requires dynamo supportc                  ó   • [        U 0 UD6$ r�   )Úflex_attention_hop)rà   rá   s     r!   Ú_flex_attention_hop_wrapperÚ3flex_attention.<locals>._flex_attention_hop_wrapper¸  s   € Ü! 4Ð2¨6Ñ2Ð2r#   )ÚbackendÚ	fullgraph)%r   r  r#  r:  r»   r  r�   r  r§   rŽ   r  rÔ   r—   r–   rÿ   r  r²   ru   rB  Úsqrtr’   r"   ÚFutureWarningr  r&   rN   r   r   Úis_dynamo_compilingÚ_dynamoÚmark_staticrL  r  Ú%_FLEX_ATTENTION_DISABLE_COMPILE_DEBUGÚis_dynamo_supportedr   rü  )r  r  r…  rù  ræ  r;  r<  r  r  r  ÚHqÚHkvÚblock_mask_q_lenÚblock_mask_kv_lenrH  r1  r·   rS   rT   rM  rO  Úflex_fns                         r!   r   r   ¼  sI  € ô^ ˜ U¸$Ò?Ü˜ EÔ*Ü�U Ô'Ü,¨U¸Ó?Ñ€E�Ø‡y�yƒ{�aÓ˜3Ÿ7™7›9¨›>¨U¯Y©Y«[¸AÓ-=Ü!Ð"QÓRÐRÞ˜EŸJ™J r›N¨c¯h©h°r«lÓ:ÜðØŸ*™* R›.Ð)¨°3·8±8¸B³<°.ð A3ð4ó
ð 	
ö
 Ø�Z‰Z˜‹]ˆØ�h‰h�q‹kˆØ‰8�q‹=ÜðØ ˜T ¨3¨%¨qð2óð ð ‡z�z�!ƒ}˜Ÿ™ ›Ó#ØÑÜð/à/4¯z©z¸!«}¨o¸ZÈÏÉÐQRËÀ}ÐTUðWóð ð ×#Ñ#×(Ñ(¨Ó+¨u¯z©z¸!«}Ó<Üðà#Ÿj™j¨›m˜_¨F°3·8±8¸A³;°-¸Èz×OgÑOg×OlÑOlÐmnÓOoÐNpÐpqðsóð ð ÑÜˆ	àÑÜ-¨eÓ9ˆ
ô ˆz˜: tÓ,Ô0FÒFÜÐHÓIÐIð 	×Ñ˜aÑ Ô$<Ó<Ø×!Ñ! !Ñ$Ô(@Ó@ñ 	à%×+Ñ+¨BÑ/ÐØ&×,Ñ,¨RÑ0ÐØ�:‰:�b‹>Ð,Ó,°·±¸³Ð?PÓ0PÜØ>¸z×?OÑ?OÐ>PÐP_Ð`e×`jÑ`jÐkmÓ`nÐ_oÐo{Ð|÷  }Eñ  }Eð  FHó  }Ið  |Jð JKð Kóð ð
 �J‰J�r‹NÐ-Ó-°#·(±(¸2³,ÐBSÓ2SØ�j‰j˜‹nÐ 0Ó0°S·X±X¸b³\ÐDUÓ5UÜØ>¸z×?OÑ?OÐ>PÐP_Ð`e×`jÑ`jÐkmÓ`nÐ_oÐo{Ð|÷  }Eñ  }Eð  FHó  }Ið  |Jð Juð uóð ð �:‰:�b‹>Ð-Ó-Ü Ø" 5§:¡:¨b£>Ð"2Ð2IÐJZÐI[Ð[\Ð]óð ð �8‰8�B‹<Ð,Ó,Ü Ø  §¡¨"£ Ð.FÐGXÐFYÐYZÐ[óð ð �}Ø”d—i’i §
¡
¨2£Ó/Ñ/ˆà‡|�|�z×/Ñ/×6Ñ6Ó6ÜðØ—|‘|�n E¨*×*BÑ*B×*IÑ*IÐ)JÈ!ðMó
ð 	
ö �jÑ,Üð\ó
ð 	
ö 
˜
Ñ*ÜØ#ðBä"ò		
ô +ØØØØØØó€Nðô
  Ñ%ðô ôô: ‡~�~×)Ñ)×+Ñ+à˜eÓ$ˆAÜ�M‰M×%Ñ% a¨Ô,Ü�M‰M×%Ñ% a¨Ö,ñ %ô  2ØØØØØ×ÑÓ!ØØó 
ÑˆˆS�*ñ !Ø��j¨Zñ
ð 	
÷ 1Ò0ÜØ3ðaò		
ô �=‰=×,Ñ,×.Ñ.ÜÐCÓDÐDò3ô 
Ô	  Gß0Ò0Ø1‰Gä—m’mØ+°WÈñˆGñ  'ØØØØØ×ÑÓ!ØØó 
ÑˆˆS�*÷ 
!ñ" ØˆS�*¨ñð ÷# 
!Õ	 ús   ÖAW"×"
W0)rF   r   Fr�   )NNNFFN)TrK   r?  re   rw  rB  rA  r  r   Úcollections.abcr   Úenumr   r   r   r   r   Útyping_extensionsr	   r
   r   r   Ú&torch._higher_order_ops.flex_attentionr   rL  Útorch._higher_order_ops.utilsr   Útorch._prims_commonr   Útorch.nn.attention._utilsr   Útorch.utils._pytreer   r   rV  Úsetr   rö   rM   ÚUserWarningr‹  ÚWarningr"   Ú__all__Ú_score_mod_signaturerô   r1   r'   r&   r%   rX   r{   rÕ   rL   ÚlistrN   r†   rŽ   r,   r—   r¨  r  r¸   rÂ   rÆ   rÌ   rõ   r×   rÙ   r$   r¬  r°  rÇ  r*   r+   rç  rë  r)   r(   r  r  r  r#  r:  rY  rF   r#   r!   Ú<module>rk     sK  ðò Pã Û Û Û Û Û Û Ý $Ý ß 6Ó 6ß 4ã Ý Ý WÝ ?Ý .Ý :ß 9ð& ).Ð %á›E€��S‘Ó !ð >Iñ(Øð(Ø!ð(Ø-1°'©]ð(à	õ(ò€ð   ¨°¸ÀÐ HÈ&Ð PÑQÐ Ø ¨°¸Ð?ÀÐGÑHÐ ØÐHÑI€ˆ)Ó IôC˜	¨ò CôL�ô ô%�
ô %ô	˜ô 	ð)�hð )Ð#4ô )ðJ &(Ø'(Øñ+Øð+à�#˜‘*˜c�/Ñ"ð+ð �#˜‘*˜c�/Ñ"ð+ð �D˜˜t™Ñ$Ñ$ð	+ð
 õ+ð\Øðàðð ðð ð	ð
 ðð ôðJØðJà
ðJð ðJð ð	Jð
 ôJðØðà
ðð ðð ð	ð
 ôð6 !Ð Ø"Ð ð¨ð ¸fô ð< U¨6°6¨>Ñ%:ô ð6¨&ð 6¸vô 6ð

Øð
àð
ð ð
ð ô	
ð9˜&ð 9 U¨6°3¨;Ñ%7ô 9÷%-ñ %-÷P_ ñ _ òDò5ð ,Ø,Ø!&ñ	2$Ø
ð2$ð ð	2$ð
 ˆ6�6˜D‘=Ð Ñ!õ2$ðjÐ,ð Ð1Dô ðÐ-ð Ð2Eô ð  -Ø+ñð õ	ð& 3Ø3ñØ�f˜f t™mÐ+Ñ,ðà˜‰oðð �s˜C�x‘ðð ð	ð
 ðð õð@ %)ñ0!Ø Ð#6Ñ6ð0!à
ˆT�zð0!ð ˆT�zð0!ð ð	0!ð
 ð0!ð ˜TÑ!ð0!ð õ0!ðr %)Ø(BØñRØ!ðRà
ˆT�zðRð ˆT�zðRð ð	Rð
 ðRð ˜TÑ!ðRð �e˜C ˜H‘oÑ%ðRð õRðj Fð °ð ¸Iô ð* %)ñIØðIà	ðIð ðIð ð	Ið ˜TÑ!õIðX
˜vð 
¨Fð 
¸6ð 
Àdô 
ð
˜Fð 
¨ð 
¸ð 
À4ô 
ð$8Øð8Øð8Ø'-ð8à
ˆ5�<‰<˜Ÿ™ u§|¡|Ð3Ñ4ô8ð~ .2Ø#'ØØØØ/3ðRð %)òRØðRà	ðRð ðRð $ dÑ*ð	Rð
 ˜DÑ ðRð �4‰<ðRð ðRð ðRð &¨Ñ,ðRð ˜TÑ!ðRð ˆe�F˜F�NÑ#Ñ# e¨F°IÐ,=Ñ&>Ñ>÷Rr#   