ó
    Eñi¸&  ã                   óP  • S SK r S SKJrJr  S SKrS SKJr  S SKJr  S SK	J
r
  SSKJrJrJr  SSKJr  SS	KJrJrJrJr  SS
KJrJrJrJr  SSKJrJrJrJ r J!r!J"r"J#r#  SSK$J%r%J&r&  SSK'J(r(J)r)J*r*J+r+  \(       a  SSKJ,r,  SSKJ-r-  \ R\                  " \/5      r0\RJ                  Rb                  r1\S 5       r2\" S\2\
" S5      SS9r3\" \Rh                  S\1Rh                  Rj                  S9r6\" \Rh                  \Rn                  Rq                  5       (       a  SOSS\1Rh                  Rr                  S9r:\" \Rv                  S\1Rv                  Rj                  S9r<\Rz                  " \1Rh                  5      S!SS.S jj5       r>\Rz                  " \1Rv                  5      SSSS.S  j5       r?g)"é    N)ÚTYPE_CHECKINGÚUnion)Úcounters)ÚCKGemmTemplate)Úload_kernel_templateé   )ÚconfigÚirÚlowering)ÚMMKernelInputs)Ú	loweringsÚmake_pointwiseÚmake_reductionÚtransform_args)Úautotune_select_algorithmÚExternKernelChoiceÚSymbolicGridFnÚTritonTemplate)Ú_use_cutlass_for_opÚuse_aten_gemm_kernelsÚuse_ck_gemm_templateÚuse_cpp_bmm_templateÚuse_cutlass_templateÚuse_nv_universal_gemm_templateÚuse_triton_template)ÚopsÚVé   )Ú_is_static_problemÚis_batch_stride_largest_or_zeroÚmm_argsÚuse_native_matmul)ÚChoiceCaller)ÚKernelTemplatec                ó6   • U" XS   5      U" X#S   5      -  U S4$ )NÚBLOCK_MÚBLOCK_Nr   © )ÚbÚmÚnÚmetaÚcdivs        ÚW/home/mande/repo/quber/.venv/lib/python3.13/site-packages/torch/_inductor/kernel/bmm.pyÚbmm_gridr/   -   s&   € á�˜‘OÓ$¡t¨A°I©Ó'?Ñ?ÀÀAÐFÐFó    ÚbmmÚ
triton_bmmT)ÚnameÚgridÚsourceÚ"cache_codegen_enabled_for_templatezat::bmm_out)Úop_overloadzat::_bmm_out_dtype_xpuzat::_bmm_out_dtype_cudaÚ	bmm_dtype)r3   r7   zat::baddbmm_out©Úlayoutc          
      ó~	  ^ ^• [        S T U4 5       5      (       Ga  T R                  5       S   S:X  d  UR                  5       S   S:X  aX  [        R                  " T S5      m [        R                  " US5      n[        R                  " [        R
                  " T U5      SS9$ S nS mU4S jnU" T 5      (       a0  [        R                  R                  R                  S	   nU" T U5      m U" U5      (       a/  [        R                  R                  R                  S   nU" X5      n[        T U5      (       añ  [        [        R                     " T S5      m [        [        R                     " US5      n[        T U/0 S
SSS9u  p‰[        R                  R                   (       aU  T R"                  [$        R&                  [$        R(                  4;   a'  U 4S jn
U Vs/ s H  n[+        U
5      " U5      PM     nn[+        [,        R.                  5      " U6 n[1        S5      " US5      nU$ [3        T XUS9u  pïnnm nSn[5        T U/US9nT R                  5       S	   n[6        S   SU SU SU SU 3==   S-  ss'   [8        R;                  SUUUUT R=                  5       UR=                  5       U5        [>        n0 nU(       a/  T RA                  5       RB                  S;   d   S5       e[D        nSU0n/ n/ n0 n[G        5       (       a   URI                  U5        UUURJ                  '   [M        USS9(       a,  Ub  UT R=                  5       :X  a  URI                  [N        5        URQ                  [        RR                  RU                  UUUUS95        [W        U5      u  nn[Y        T X5      nU(       aP  U(       aI  [[        X>UU5      (       a7  []        U5      (       a'  SSK/J0n  URc                  UUURe                  5       5        [g        UT U5      (       a'  SSK4J5n  URm                  UUURe                  5       5        [o        X>UU5      (       a&  [p        Rr                  " UUURe                  5       5        U(       a$  [u        X>UUT U5      (       a  SSK;J<n  U" UUU5        [{        UUURe                  5       U5      $ s  snf )zX
Lowering for autotuning aten.bmm with different backends (Aten, Triton, CUTLASS, etc.)
c              3   óZ   #   • U  H!  oR                  5       R                  S :H  v •  M#     g7f)ÚcpuN)Ú
get_deviceÚtype)Ú.0Úxs     r.   Ú	<genexpr>Útuned_bmm.<locals>.<genexpr>M   s   é € Ð
>²¨A�<‰<‹>×Ñ %Ö'²ùs   ‚)+r   r   éÿÿÿÿ)Úaxisc                 óœ   • [         R                  " U 5      (       d  g[         R                  " U SS9u  p[        U[         R                  5      $ )NTF)Úfreeze)r
   Úis_storage_and_layoutÚas_storage_and_layoutÚ
isinstanceÚFlexibleLayout)ÚtÚ_r:   s      r.   Úis_valid_to_require_contiguousÚ1tuned_bmm.<locals>.is_valid_to_require_contiguousT   s=   € Ü×+Ò+¨A×.Ñ.ØÜ×0Ò0°¸5ÑA‰IˆAÜ˜f¤b×&7Ñ&7Ó8Ð8r0   c                 ó´   • US   S:H  =(       a    U S   S:H  =(       d    US   U S   :¬  =(       d)    US   S:H  =(       a    U S   S:H  =(       d    US   U S   :¬  $ )NrD   r   éþÿÿÿr(   )ÚsizesÚstridess     r.   Ú is_preferred_layout_as_bmm_inputÚ3tuned_bmm.<locals>.is_preferred_layout_as_bmm_inputZ   sf   € ð ˜‘˜qÑ ×Q e¨B¡i°1¡n×&P¸À¹ÀuÈRÁyÑ8P÷Uà˜"‘+ Ñ"×S¨¨b©	°Q©×(R¸'À"¹+ÈÈrÉÑ:RðUr0   c                 óÔ   >• UR                   S   R                  5       nUR                   S   R                  5       nT" X#5      (       d  [        R                  R                  U 5      n U $ )NÚval)r,   ÚsizeÚstrider
   ÚExternKernelÚrequire_contiguous)rL   Úmeta_trR   rS   rT   s       €r.   Úmay_require_contiguousÚ)tuned_bmm.<locals>.may_require_contiguousd   sU   ø€ Ø—K‘K Ñ&×+Ñ+Ó-ˆEØ—k‘k %Ñ(×/Ñ/Ó1ˆGÙ3°E×CÑCÜ—O‘O×6Ñ6°qÓ9�ØˆHr0   r   TNF)ÚargsÚkwargsÚ	broadcastÚtype_promotion_kindÚconvert_input_to_boolc                 óD   >• [         R                  " U TR                  SS9$ )NF)Úuse_compute_types)r   Úto_dtypeÚdtype)rA   Úmat1s    €r.   Ú	_to_dtypeÚtuned_bmm.<locals>._to_dtype‚   s   ø€ Ü—|’| A t§z¡zÀUÑKÐKr0   Údot)r:   Ú	out_dtyper1   )rl   Úaten_mm_infoz	aten.bmm_rM   zZTuned aten.bmm: batch=%s, m=%s, n=%s, k=%s, mat1_dtype=%s, mat2_dtype=%s, output_layout=%s)ÚcudaÚxpuz+out_dtype is only supported for CUDA or XPUrl   ©Úcheck_max_autotune)Úkwarg_overrides)ÚCUTLASS3xGemmTemplate)ÚCppBmmTemplate)Úadd_nv_universal_gemm_choices)>ÚallÚget_sizeÚLÚ	unsqueezeÚsum_Úmulr   ÚgraphÚcurrent_noder_   r"   r   Úatenr   Úinductor_configÚtritonÚcodegen_upcast_to_fp32rg   ÚtorchÚfloat16Úbfloat16r   r   rk   r   r!   r   r   ÚlogÚinfoÚ	get_dtypeÚaten_bmmr>   r?   Úaten_bmm_dtyper   ÚappendÚuidr   Úbmm_templateÚextendÚchoicesÚget_template_configsr   r    r   r   Úcodegen.cutlass.gemm_templaters   Úadd_cutlass_gemm_choicesÚnodesr   Úcodegen.cpp_bmm_templatert   Úadd_choicesr   r   Úadd_ck_gemm_choicesr   Úcodegen.nv_universal_gemmru   r   ) rh   Úmat2rl   r:   rN   r]   Ú	meta_mat1Ú	meta_mat2r_   r`   ri   rA   Úmul_pointwiseÚdot_reductionr*   r+   Úkr3   Úkernel_inputsÚ
batch_sizeÚaten_handlerÚaten_extra_kwargsrŽ   Útemplates_to_userr   rM   Ú
is_nonzeroÚbatch_stride_largest_or_zerors   rt   ru   rT   s    `                              @r.   Ú	tuned_bmmr¤   H   sQ  ù€ ô
 Ñ
>°$¸±Ó
>×>Ò>à�=‰=‹?˜1Ñ Ó" d§m¡m£o°aÑ&8¸AÓ&=Ü—;’;˜t RÓ(ˆDÜ—;’;˜t QÓ'ˆDÜ—6’6œ!Ÿ%š%  dÓ+°!Ñ4Ð4ò	9ò	Uõ	ñ *¨$×/Ñ/ÜŸ™×,Ñ,×1Ñ1°!Ñ4ˆIÙ)¨$°	Ó:ˆDÙ)¨$×/Ñ/ÜŸ™×,Ñ,×1Ñ1°!Ñ4ˆIÙ)¨$Ó:ˆDä˜˜t×$Ñ$ÜœŸ™Ò(¨¨rÓ2ˆÜœŸ™Ò(¨¨qÓ1ˆÜ%Ø˜�ØØØ $Ø"'ñ
‰ˆô ×!Ñ!×8×8¸T¿Z¹ZÜ�M‰MÜ�N‰NðL
ó >
õ
Lñ ;?Ó?º$°Q”N 9Ô-¨aÖ0¹$ˆDÐ?ä&¤s§w¡wÔ/°Ð6ˆÜ& uÔ-¨m¸QÓ?ˆàÐô #*Øˆd¨Yñ#Ñ€Aˆ!ˆV�T˜4ð €Dô # D¨$ <¸9ÑE€Mð —‘“ Ñ#€JÜˆ^Ñ˜y¨¨°A°a°S¸¸!¸¸A¸a¸SÐAÓBÀaÑGÓBÜ‡H�HØdØØ	Ø	Ø	Ø�‰ÓØ�‰ÓØô	ô (0€LØÐÞØ�‰Ó ×%Ñ%¨Ó8ð 	
Ø9ó	
Ð8ô &ˆØ(¨)Ð4Ðà"$€Gð IKÐØ€Oä×ÑØ×Ñ Ô-Ø,=ˆ˜×(Ñ(Ñ)ä˜6°e×<ØÑ˜Y¨$¯.©.Ó*:Ó:ð 	×Ñ¤Ô-ð ‡N�NÜ	�	‰	×&Ñ&ØØØØ+ð	 	'ð 	
ôô ' vÓ.�M€A€zÜ#BÀ4ÈÓ#VÐ æ$ÞÜ  ¨A¨q×1Ñ1Ü ×%Ñ%åIà×6Ñ6Ø�V˜]×0Ñ0Ó2ô	
ô ˜F D¨$×/Ñ/Ý=à×"Ñ"ØØØ×ÑÓ!ô	
ô ˜F q¨!×,Ñ,Ü×*Ò*¨7°F¸M×<OÑ<OÓ<QÔRæÔ4°VÀÀ1ÀdÈD×QÑQÝMá% g¨v°}ÔEä$ T¨7°M×4GÑ4GÓ4IÈ6ÓRÐRùò @s   ÇR:)ÚalphaÚbetar:   c                ó”  • [        X5      (       a…  US:X  a  SnO[        [        R                     " X@5      nUS:X  a  SnO9[        [        R                     " U[        [        R                     " X5      5      n[        [        R
                     " Xg5      $ [        XXS9u  p‰p¥pn [        XU/[        X4S9S9nUR                  5       S   n[        S   SU SU SU	 SU
 3==   S-  ss'   [        R                  S	UUU	U
UR                  5       UR                  5       U R                  5       U5	        S
n/ n/ n[        5       (       a  UR                  [         5        [#        USS9(       a  UR                  [$        5        UR'                  [(        R*                  R-                  X¿U5      5        [/        XÞUR1                  5       U5      $ )zW
Lowering for autotuning aten.mm with different backends (Aten, Triton, CUTLASS, etc.)
r   r9   )r¥   r¦   )Úscalarsrm   zaten.baddbmm_rM   r   zkTuned aten.baddbmm: batch_size=%s, m=%s, n=%s, k=%s, mat1_dtype=%s, mat2_dtype=%s, inp=%s, output_layout=%sÚbaddbmmFrp   )r"   r   r~   r{   r1   Úaddr!   r   Údictrw   r   r…   r†   r‡   r   rŠ   Úaten_baddbmmr   rŒ   r�   r   rŽ   r�   r   r’   )Úinprh   r—   r¥   r¦   r:   Úarg1Úarg2r*   r+   rœ   r�   rž   r3   rŽ   r¡   s                   r.   Útuned_baddbmmr°   ç   sŸ  € ô
 ˜×$Ñ$Ø�1‹9Ø‰DäœTŸX™XÒ& tÓ1ˆDà�A‹:Ø‰DäœTŸX™XÒ& u¬i¼¿¹Ò.AÀ$Ó.MÓNˆDäœŸ™Ò" 4Ó.Ð.ô (/¨t¸3Ñ'NÑ$€Aˆ!�T ô #Ø	�DÐ¤4¨eÑ#?ñ€Mð
 —‘“ Ñ#€JÜˆ^Ñ˜}¨Z¨L¸¸!¸¸A¸a¸SÀÀ!ÀÐEÓFÈ!ÑKÓFÜ‡H�HØuØØ	Ø	Ø	Ø�‰ÓØ�‰ÓØ�‰‹Øô
ð €Dà"$€Gð IKÐÜ×ÑØ×Ñ¤Ô-ä˜6°e×<Ø×Ñ¤Ô-ð ‡N�NÜ	�	‰	×&Ñ& }ÈÓMôô % T°M×4GÑ4GÓ4IÈ6ÓRÐRr0   )N)@ÚloggingÚtypingr   r   r‚   Útorch._dynamo.utilsr   Ú7torch._inductor.codegen.rocm.ck_universal_gemm_templater   Ú torch._inductor.kernel.mm_commonr   Ú r	   r   r
   r   rx   r�   r   r   r   r   r   Úselect_algorithmr   r   r   r   Úutilsr   r   r   r   r   r   r   Úvirtualizedr   r   Ú	mm_commonr   r    r!   r"   r#   r$   Ú	getLoggerÚ__name__r…   r~   r/   rŒ   r1   Úoutrˆ   ro   Úis_availableÚ	dtype_outr‰   r©   r¬   Úregister_loweringr¤   r°   r(   r0   r.   Ú<module>rÁ      s~  ðã ß 'ã Ý (Ý RÝ Aç ;Ñ ;Ý *ß PÓ P÷ó ÷÷ ñ ÷ !÷ó ö Ý!Ý1à×Ò˜Ó!€Ø‡y�y‡~�~€ð ñGó ðGñ Ø	Ø	Ù Ó-Ø'+ñ	€ñ ˜eŸi™i¨ÀDÇHÁHÇLÁLÑQ€Ù#Ø	‡I�IØ %§	¡	× 6Ñ 6× 8Ñ 8ÑÐ>WØ	Ø—‘×"Ñ"ñ	€ñ "Ø	‡M�MÐ$°$·,±,×2BÑ2Bñ€ð
 ×Ò�T—X‘XÓð[S°Dõ [Só ð[Sð| ×Ò�T—\‘\Ó"Ø,-°A¸dô 8Só #ñ8Sr0   