ó
    Eñiî ã                  óú  • S SK Jr  S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
r
S SKJr  S SKJrJrJrJrJrJr  S SKJr  S SKrS SKrS SKrS SKJr  S SKJr  S SKJr  S S	KJr  S S
K J!r!  S SK"J#r#  S SK$J%r%J&r&J'r'  S SK(J)r)J*r*J+r+J,r,  SSK-J.r.  SSK/J0r0J1r1J2r2  SSK3J4r4  SSK5J6r6J7r7  SSK8J9r9J:r:J;r;  \(       a  S SK<J=r=  SSK1J>r>  SSK?J@r@  SSKAJBrB  SSKCJDrD  SSKEJFrFJGrGJHrH  SSK2JIrIJJrJJKrK  SSKLJMrMJNrNJOrOJPrPJQrQJRrRJSrSJTrTJUrUJVrV  SSKWJXrXJYrYJZrZ  SSK[J\r\  SS K]J^r^J_r_J`r`Jara  SS!KbJcrcJdrd  SS"KeJfrfJgrgJhrhJiriJjrj  \(       a  S S#K<JkrkJlrlJmrm  S S$KJnrn  \RÞ                  " \p5      rq\Rä                  Rç                  \pS%5      rt\Rä                  Rç                  \pS&5      ru\Rä                  Rç                  \pS'5      rv\a" 5       Rî                  rx\#" / S(Q5      rySASBS) jjrz\Rö                   " S* S+5      5       r| " S, S-\|5      r} " S. S/\|5      r~SCS0 jr\" S1\^\^S29r€\Rö                   " S3 S45      5       r� " S5 S6\5      r‚ " S7 S8\`\€   \\€   5      rƒ " S9 S:\J5      r„\Rö                  " S;S<9 " S= S>5      5       r… " S? S@\†5      r‡g)Dé    )ÚannotationsN)ÚCounter)ÚAnyÚGenericÚ
NamedTupleÚOptionalÚTYPE_CHECKINGÚUnion)ÚTypeVar)Úmetrics)ÚMultiTemplateBuffer)Úanalyze_memory_coalescing)Úfree_unbacked_symbols)Úimmutable_dict)Ú
OrderedSet)ÚFloorDivÚIdentityÚModularIndexing)Úfree_symbol_is_typeÚ
prefix_strÚsymbol_is_typeÚSymTé   )Úcountersé   )ÚconfigÚirÚ	scheduler)Úprologue_preserves_zero_mask)Ú	code_hashÚPyCodeCache)Ú	MemoryDepÚStarDepÚWeakDep)ÚCallable©ÚIRNode)Ú!indexing_dtype_strength_reduction)ÚCoordescTuner)ÚDeviceProperties)Ú
green_textÚlast_power_of_2Úyellow_text)ÚBaseSchedulerNodeÚBaseSchedulingÚ	WhyNoFuse)
Úcache_property_on_selfÚexpr_fits_within_32bitÚget_dtype_sizeÚIndentedBufferÚPlaceholderÚprefix_is_reductionÚsympy_index_symbolÚsympy_productÚ
sympy_subsÚunique)ÚopsÚ
OpsWrapperÚVé   )ÚBlockPatternMatcher)ÚCSEVariableÚindex_prevent_reorderingÚKernelÚPythonPrinter)ÚMultiKernelÚSizeHintMultiKernel)ÚDisableReductionÚEnableReductionÚNodeScheduleEntryÚNodeScheduleMarkerÚSIMDKernelFeatures)ÚIterableÚIteratorÚSequence)ÚCoalesceVarAnalysisÚ
perf_hintsÚscheduleÚfusion)ÚzÚyÚxÚr0_Úr1_c                ól   • [         R                  R                  R                  R                  nUb  U$ U $ ©N)ÚtorchÚ	_inductorr   ÚtritonÚ	max_tiles)Údefaultr\   s     ÚY/home/mande/repo/quber/.venv/lib/python3.13/site-packages/torch/_inductor/codegen/simd.pyÚget_max_tilesr_   ^   s-   € Ü—‘×&Ñ&×-Ñ-×7Ñ7€IØ!Ñ-ˆ9Ð:°7Ð:ó    c                  óî   ^ • \ rS rSrSr\R                  R                  \R                  R                  S.               S	U 4S jjjr\	\
S
S j5       5       rSS jr\	\
SS j5       5       rSrU =r$ )ÚIterationRangeséc   a”  
Each range tree represents multiple sets of iteration indexing
in a single tiled dimension in the output kernel.

If you have two loops ranges one (4, 3, 2) and another (4, 6),
then the range tree will be:
        4 (i0)
    3 (i1)  6 (i3)
    2 (i2)
Where i0 is shared between both loops, but then the split into
different indexing vars.  All loop ranges must iterate over
the same number of elements.
)ÚdivisorÚlengthc               óŽ   >• [         T
U ]  5         Xl        X l        X0l        X@l        XPl        Xpl        X€l        X`l	        X�l
        g rX   )ÚsuperÚ__init__ÚnameÚvar_listÚ
var_rangesÚnumelÚprefixrd   re   ÚkernelÚroot)Úselfri   rj   rk   rl   rm   rn   rd   re   ro   Ú	__class__s             €r^   rh   ÚIterationRanges.__init__s   s=   ø€ ô 	‰ÑÔØŒ	Ø ŒØ$ŒØŒ
ØŒØŒØŒØŒØ�	r`   c                ó,   • [        U R                  5      $ rX   )r6   rm   ©rp   s    r^   Úis_reductionÚIterationRanges.is_reduction‹   s   € ô # 4§;¡;Ó/Ð/r`   c                ó,   • [        U R                  5      $ rX   )r7   ri   rt   s    r^   ÚsymbolÚIterationRanges.symbol�   s   € Ü! $§)¡)Ó,Ð,r`   c                ó|   • [         R                  " 5        VVs0 s H  u  pX!_M	     nnnX0R                     $ s  snnf rX   )r   Úitemsrm   )rp   Úsymtrm   Úprefix_to_symts       r^   r|   ÚIterationRanges.symt“   s;   € ô <F×;KÒ;KÔ;MÔNÒ;M©<¨4˜&š,Ñ;MˆÑNØŸk™kÑ*Ð*ùó Os   ™8)	rd   rn   re   ri   rl   rm   ro   rj   rk   )ri   Ústrrj   úlist[sympy.Symbol]rk   údict[sympy.Symbol, sympy.Expr]rl   ú
sympy.Exprrm   r   rn   Ú
SIMDKernelro   ÚIterationRangesRootÚreturnÚNone©r…   Úbool©r…   zsympy.Symbol)r…   r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚsympyÚSÚOnerh   Úpropertyr1   ru   rx   r|   Ú__static_attributes__Ú__classcell__©rq   s   @r^   rb   rb   c   sµ   ø† ñð. —‘—‘Ø�w‰w�{‰{ñàðð %ðð 3ð	ð
 ðð ðð ðð "ðð 
÷ð ð0 Øó0ó ó ð0ô-ð Øó+ó ó ö+r`   rb   c                  ó¶   ^ • \ rS rSrSr S                     SU 4S jjjrSS jrSS jrSS jrSS jr	    SS jr
SS	 jr    SS
 jrSrU =r$ )r„   éš   z­
Root of a iteration range tree that represents a single
tiled dimension in the output kernel. It contains multiple
sets of iteration represented with IterationRangesEntry.
c          
     óÆ   >• Uc  0 n[         TU ]  U/ 0 UUUU S9  X@l        0 U l        X`l        U(       a  U R
                  (       a  U	b   eXpl        X€l        X�l        X l	        g )N)ri   rj   rk   rl   rm   rn   ro   )
rg   rh   ÚindexÚnodesÚ	pid_cacheru   Úis_loopÚ
tensor_dimÚgrid_dimÚhas_zdim)rp   ri   rl   rm   r™   rn   r›   rœ   r�   rž   rŸ   rq   s              €r^   rh   ÚIterationRangesRoot.__init__¡   sx   ø€ ð ÑØˆIÜ‰ÑØØØØØØØð 	ñ 	
ð Œ
à=?ˆŒ
ð *3Œö
 ˜t×0×0°XÑ5EÐFÐFØŒà$Œà ŒØ �r`   c                ó>   • SU R                   < SU R                   S3$ )NzIterationRangesRoot(ú, z, ...))ri   rl   rt   s    r^   Ú__repr__ÚIterationRangesRoot.__repr__Ì   s   € Ø% d§i¡i¡]°"°T·Z±Z°LÀÐGÐGr`   c                óf   • U R                   R                  5        H  nUR                  5         M     g rX   )rš   ÚvaluesÚcache_clear)rp   Únodes     r^   r§   ÚIterationRangesRoot.cache_clearÏ   s%   € Ø—J‘J×%Ñ%Ö'ˆDØ×ÑÖò (r`   c                ó2   • [        U R                   S35      $ )Nr™   )r7   rm   rt   s    r^   Ú	index_symÚIterationRangesRoot.index_symÓ   s   € Ü! T§[¡[ M°Ð"7Ó8Ð8r`   c                ó–  • [         R                  R                  R                  X-  U R                  5      (       a  [        U R                  5       U5      nO[        U R                  5       X5      nX0R                  ;  a¼  [        U R                   [        [         R                  R                  5       3UUUU 5      nU[         R                  R                  UR                  5       '   U R                   R#                  UR                  5       5        X R$                  UR                  5       '   X@R                  U'   U R                  U   $ )z6
Lookup a given RangeTreeEntry, creating it if needed
)r=   ÚgraphÚsizevarsÚstatically_known_equalsrl   r   r«   r   rš   ÚIterationRangesEntryrm   Únextrn   Úiter_vars_countÚrange_tree_nodesrx   rj   Úappendrk   )rp   rd   re   Úexprr¨   s        r^   ÚlookupÚIterationRangesRoot.lookupÖ   sï   € ô �7‰7×Ñ×3Ñ3°GÑ4DÀdÇjÁj×QÑQÜ˜DŸN™NÓ,¨gÓ6‰Dä" 4§>¡>Ó#3°WÓEˆDà—z‘zÓ!Ü'Ø—;‘;�-¤¤Q§X¡X×%=Ñ%=Ó >Ð?Ð@ØØØØóˆDð 8<ŒA�H‰H×%Ñ% d§k¡k£mÑ4Ø�M‰M× Ñ  §¡£Ô/Ø-3�O‰O˜DŸK™K›MÑ*Ø#�J‰J�tÑØ�z‰z˜$ÑÐr`   c                óÀ   • [         R                  R                  n/ n[        U5       H'  nUR	                  U R                  X$5      5        X$-  nM)     / [        U5      Q$ rX   )r�   r�   r‘   Úreversedrµ   r·   )rp   Úlengthsrd   Úitervarsre   s        r^   Úconstruct_entriesÚ%IterationRangesRoot.construct_entriesí   sT   € ô —'‘'—+‘+ˆØˆÜ˜wÖ'ˆFØ�O‰O˜DŸK™K¨Ó8Ô9ØÑ&ŠGñ (ð %”˜(Ó#Ð$Ð$r`   c                ój   • U R                  U5       Vs/ s H  o"R                  5       PM     sn$ s  snf rX   )r½   rx   )rp   r»   Úes      r^   Ú	constructÚIterationRangesRoot.construct÷   s+   € Ø$(×$:Ñ$:¸7Ô$CÓDÒ$C˜q—‘–
Ñ$CÑDÐDùÒDs   ”0c           
     óŽ  ^^^	^
• SS jmUR                    Vs/ s H,  n[        R                  R                  R	                  U5      PM.     nnU Vs/ s H)  oD(       d  M  UR
                  U R
                  :X  d  M'  UPM+     nnUR                  U4S jS9  [        R                  R                  m/ m	/ m
UU	U
4S jnU H|  n[        R                  R                  R                  UR                  T5      (       d8  U" U R                  T[        UR                  T5      5      5        UR                  mU" U5        M~     [        R                  R                  R                  U R                   T5      (       d,  U" U R                  T[        U R                   T5      5      5        / [#        T	5      Q/ [#        T
5      Q4$ s  snf s  snf )z,Figure out vars from this tree used in indexc                óä   • [         R                  R                  R                  U R                  5      n[         R                  R                  R                  U R
                  5      S:H  nX(       + 4$ )zò
Gets the key for sorting nodes. When two nodes have the
same divisor, the node with length as 1 should be handled
first so the current divisor is not changed after multiplied
node.length. Returns `not length_is_one_hint` for ascending
sort.
r>   )r=   r®   r¯   Úoptimization_hintrd   re   )rT   Údivisor_hintÚlength_is_one_hints      r^   Úget_sort_keyÚ8IterationRangesRoot.vars_and_sizes.<locals>.get_sort_keyÿ   sS   € ô Ÿ7™7×+Ñ+×=Ñ=¸a¿i¹iÓHˆLÜ!"§¡×!1Ñ!1×!CÑ!CÀAÇHÁHÓ!MÐQRÑ!RÐØ Ô"8Ð9Ð9r`   c                ó   >• T" U 5      $ rX   © )rT   rÈ   s    €r^   Ú<lambda>Ú4IterationRangesRoot.vars_and_sizes.<locals>.<lambda>  s	   ø€ ¡¨a¤r`   ©Úkeyc                ó˜   >• TR                  U R                  5       5        TR                  U R                  5        TU R                  -  mg rX   )rµ   rx   re   )r¨   rd   Ú
index_varsÚsizess    €€€r^   ÚaddÚ/IterationRangesRoot.vars_and_sizes.<locals>.add  s5   ø€ à×Ñ˜dŸk™k›mÔ,Ø�L‰L˜Ÿ™Ô%Ø §¡Ñ+‰Gr`   )rT   r±   r…   ztuple[int, bool])Úfree_symbolsr=   rn   r´   Úgetrm   Úsortr�   r�   r‘   r®   r¯   r°   rd   r·   r   rl   rº   )rp   r™   Úsrš   ÚnrÓ   r¨   rd   rÈ   rÑ   rÒ   s          @@@@r^   Úvars_and_sizesÚ"IterationRangesRoot.vars_and_sizesú   sR  û€ ô

	:ð <A×;MÒ;MÓNÒ;M°a”—‘×*Ñ*×.Ñ.¨qÖ1Ñ;MˆÐNÙ!ÓCšE�q£Q“¨1¯8©8°t·{±{Ñ+B—™EˆÐCØ�
‰
Ô0ˆ
Ñ1Ü—'‘'—+‘+ˆØˆ
Øˆ÷	,ó ˆDÜ—7‘7×#Ñ#×;Ñ;¸D¿L¹LÈ'×RÑRá�D—K‘K ¬°$·,±,ÀÓ)HÓIÔJØŸ,™,�Ù�ŽIñ ô �w‰w×Ñ×7Ñ7¸¿
¹
ÀG×LÑLá�—‘˜G¤X¨d¯j©j¸'Ó%BÓCÔDà&”˜*Ó%Ð&Ð(:¬(°5«/Ð(:Ð:Ð:ùò/ OùÚCs   ˜3F=Á
GÁGÁ;G)rž   rŸ   r™   rœ   rš   r›   r�   rX   )ri   r   rl   r‚   rm   r   r™   Úintrn   rƒ   r›   úOptional[dict[str, str]]rœ   rˆ   r�   úOptional[int]rž   rÞ   rŸ   rˆ   r…   r†   ©r…   r   ©r…   r†   r‰   )rd   r‚   re   r‚   r…   r±   )r»   úlist[sympy.Expr]r…   zlist[IterationRangesEntry])r»   rá   r…   r€   )r™   r‚   r…   z+tuple[list[sympy.Symbol], list[sympy.Expr]])rŠ   r‹   rŒ   r�   rŽ   rh   r£   r§   r«   r·   r½   rÁ   rÚ   r“   r”   r•   s   @r^   r„   r„   š   sÎ   ø† ñð /3ð)!àð)!ð ð)!ð ð	)!ð
 ð)!ð ð)!ð ,ð)!ð ð)!ð "ð)!ð  ð)!ð ð)!ð 
÷)!ð )!ôVHôô9ô ð.%Ø'ð%à	#ô%ôEð(;Øð(;à	4÷(;ò (;r`   r„   c                  óŠ   ^ • \ rS rSr            SU 4S jjrSS jrSS jrSS jrSS jrSS jr	SS jr
SS	 jrS
rU =r$ )r±   i%  c                ó  >• [         TU ]  UUR                  U-  UR                  UR                  UR
                  UUUR                  UR                  S9	  XPl        [        R                  " S 5      " U R                  5      U l        X@l        g )N)	ri   rl   rj   rk   rm   rd   re   rn   ro   )rg   rh   rl   rj   rk   rm   rn   ro   ÚparentÚ	functoolsÚ	lru_cacheÚ_codegenÚcodegenr¶   )rp   ri   rd   re   r¶   rä   rq   s         €r^   rh   ÚIterationRangesEntry.__init__&  sx   ø€ ô 	‰ÑØØ—,‘, Ñ'Ø—_‘_Ø×(Ñ(Ø—=‘=ØØØ—=‘=Ø—‘ð 	ñ 
	
ð ŒÜ ×*Ò*¨4Ô0°·±Ó?ˆŒØ�	r`   c                óŠ   • SU R                    SU R                   SU R                   SU R                   SU R                   S3$ )NzIterationRangesEntry(r¢   Ú))ri   rd   re   r¶   rk   rt   s    r^   r£   ÚIterationRangesEntry.__repr__=  sH   € Ø& t§y¡y k°°D·L±L°>ÀÀDÇKÁKÀ=ÐPRÐSW×S\ÑS\ÐR]Ð]_Ð`d×`oÑ`oÐ_pÐpqÐrÐrr`   c                óN   ^• U4S jU l         S U R                   l        TU l        g )Nc                 ó   >• T $ rX   rË   )ri   s   €r^   rÌ   Ú/IterationRangesEntry.set_name.<locals>.<lambda>A  s   ø€ ™tr`   c                 ó   • g rX   rË   rË   r`   r^   rÌ   rï   B  s   € ¨4r`   )rè   r§   ri   )rp   ri   s    `r^   Úset_nameÚIterationRangesEntry.set_name@  s   ø€ Ü#ˆŒÙ#/ˆ�‰Ô Øˆ�	r`   c                ó8   • U R                   R                  5         g rX   )rè   r§   rt   s    r^   r§   Ú IterationRangesEntry.cache_clearE  s   € Ø�‰× Ñ Õ"r`   c                óX   • [         R                  R                  U 5        U R                  $ rX   )r=   rn   Úcodegen_iteration_ranges_entryri   rt   s    r^   rç   ÚIterationRangesEntry._codegenH  s   € Ü	�‰×/Ñ/°Ô5Ø�y‰yÐr`   c                ó  • / n[        U R                  [        R                  5      (       a  U$ [        U R                  [        [
        45      (       d   [        U R                  5      5       eU R                  R                  SS   H{  n[        U[        R                  [        R                  45      (       a  M4  UR                  n[        U5      S:”  d  MQ  [        S U 5       5      (       d  Mj  UR                  U5        M}     U$ )Nr>   r   c              3  óV   #   • U  H  n[        U[        R                  5      v •  M!     g 7frX   )r   r   ÚSIZE©Ú.0rØ   s     r^   Ú	<genexpr>Ú8IterationRangesEntry.precomputed_args.<locals>.<genexpr>U  s!   é € ð ,Ú:A°Q”N 1¤d§i¡i×0Ð0º'ùs   ‚'))Ú
isinstancer¶   r�   ÚSymbolr   r   ÚtypeÚargsÚIntegerrÕ   ÚlenÚallrµ   )rp   Úprecomputed_argsÚargÚsymbolss       r^   r  Ú%IterationRangesEntry.precomputed_argsL  sÇ   € à-/ÐÜ�d—i‘i¤§¡×.Ñ.Ø#Ð#Ü˜$Ÿ)™)¤h´Ð%@×AÑAÐRÄ4ÈÏ	É	Ã?ÓRÐAØ—9‘9—>‘> ! "Ó%ˆCÜ˜c¤E§M¡M´5·<±<Ð#@×AÓAØ×*Ñ*�Ü�w“< !Õ#¬ñ ,Ù:Aó,÷ )ó )ð %×+Ñ+¨CÖ0ñ &ð  Ðr`   c                ó,   • [        U R                  5      $ rX   )Úhashri   rt   s    r^   Ú__hash__ÚIterationRangesEntry.__hash__[  s   € Ü�D—I‘I‹Ðr`   c                ób   • [        U[        5      (       d   eU R                  UR                  :H  $ rX   )rÿ   r±   ri   )rp   Úothers     r^   Ú__eq__ÚIterationRangesEntry.__eq__^  s)   € Ü˜%Ô!5×6Ñ6Ð6Ð6Ø�y‰y˜EŸJ™JÑ&Ð&r`   )rè   r¶   ri   rä   )ri   r   rd   r‚   re   r‚   r¶   r‚   rä   rb   r…   r†   rß   )ri   r   r…   r†   rà   )r…   rá   ©r…   rÜ   )r  Úobjectr…   rˆ   )rŠ   r‹   rŒ   r�   rh   r£   rñ   r§   rç   r  r  r  r“   r”   r•   s   @r^   r±   r±   %  sk   ø† ðàðð ðð ð	ð
 ðð  ðð 
÷ô.sôô
#ôô ô÷'ò 'r`   r±   c                ó�   • U [        S5      :X  a  gU [        S5      :X  a  g[        R                  " U 5      (       a  g[        U 5      $ )NÚinfzfloat("inf")z-infzfloat("-inf")zfloat("nan"))ÚfloatÚmathÚisnanÚrepr)Úvalues    r^   Úconstant_reprr  c  s<   € Ø”�e“ÓØØ	”%˜“-Ó	ØÜ	�Š�E×	Ñ	ØÜ�‹;Ðr`   ÚCSEVariableType)Úboundr]   c                  ó4   • \ rS rSr% S\S'   S\S'   S\S'   Srg)	ÚPartialAccumulateip  r   Úbuffer_nameÚreduction_typer   r  rË   N)rŠ   r‹   rŒ   r�   Ú__annotations__r“   rË   r`   r^   r  r  p  s   ‡ àÓØÓØ†Jr`   r  c                  óV   • \ rS rSr% SrS\S'   S\S'   S\S'   S\S	'   S
\S'   S\S'   Srg)ÚNodeInfoiw  z>
Pre-computed node information for combo kernel partitioning.
ÚlistÚnode_scheduleÚdictÚtilingr   rl   ÚrnumelrJ   Úfeaturesrˆ   Úis_persistent_reductionrË   N)rŠ   r‹   rŒ   r�   rŽ   r"  r“   rË   r`   r^   r$  r$  w  s*   ‡ ñð ÓØƒLØƒJØƒKØ Ó Ø!Ö!r`   r$  c                  óÄ  ^ • \ rS rSr% Sr\rS\S'   S\S'   SrS\S'   S	\S
'        S?               S@U 4S jjjr	    SAS jr
SBS jrS r\\SCS j5       5       rSDS jrSES jr\SFS j5       rSGS jr            SHS jrSIS jrSJS jrSKS jrSGS jrSGS jrSLS jrSCS jrSBS jrSMS jrSFS jrSFS jrSNS  jr       SOS! jr!      SOS" jr"SPS# jr#SQS$ jr$\%      SRS% j5       r&\'\(RR                  RT                  4       SSS& jj5       r+\'\(RR                  RT                  4       STS' jj5       r,    SUS( jr-\'      SVS) j5       r.SWS* jr/SWS+ jr0SXS, jr1    SNS- jr2SYS. jr3SZS/ jr4S[S0 jr5S1 r6 S\       S]S2 jjr7\8Rr                        S^S3 j5       r:S_S4 jr;\%S5 5       r<S`S6 jr=S7 r>S8 r?S9 r@S: rAS; rBS< rCSaS= jrDS>rEU =rF$ )brƒ   i„  zg
Common base class for Triton/Halide codegen which both use flattened indexing rather than loop nests.
zCallable[[sympy.Expr], str]ÚsexprÚkexprFrˆ   Úallow_block_ptrr   Úkernel_namec                ó¸  >^ • Uc  0 n[         TT ]  5         UT l        UR                  5       T l        [        5       T l        [        5       T l        UR                  5        VV	s0 s H/  u  p‰U[        R                  R                  R                  U	5      _M1     sn	nT l        / T l        0 T l        [         R"                  " 5       T l        UR'                  5       T l        Ub  UOT R+                  5       T l        UT l        UT l        Ub  UOT R3                  5       T l        UT l        T R9                  5       T l        S T l        [         R"                  " 5       T l        ST l         [B        RD                  RF                  (       a�  T R                  RH                   Hv  n
[K        U
[L        RN                  5      (       d  M$  [K        U
RP                  [R        RT                  5      (       d  MO  U
RP                  RW                  5       S:X  d  Mo  ST l           O   [X        RZ                  SU 4S jj5       nUT l.        T R_                  U5        ST l0        / T l1        g s  sn	nf )NFÚdotTc                óÞ   >• [         R                  R                  R                  U TR	                  5       5      n TR
                   H  nTR                  X5      n M     TR                  U 5      $ rX   )r=   r®   r¯   Úsimplify_with_rangesrk   Úrange_treesÚcombine_contiguous_dimsÚcombine_modular_indexing_pairs)r™   Útreerp   s     €r^   Úsimplify_indexingÚ.SIMDKernel.__init__.<locals>.simplify_indexingÄ  sY   ø€ ä—G‘G×$Ñ$×9Ñ9¸%ÀÇÁÓARÓSˆEØ×(Ô(�Ø×4Ñ4°UÓA’ñ )ð ×6Ñ6°uÓ=Ð=r`   r   )r™   r‚   )2rg   rh   r*  Úget_mutationsÚ	mutationsr4   ÚbodyÚindexing_coder{   r=   r®   r¯   ÚsimplifyÚnumelsr5  r´   Ú	itertoolsÚcountr³   ru   Úinside_reductionÚ should_use_cooperative_reductionÚcooperative_reductionÚtiling_scoresr(  Úshould_use_persistent_reductionÚpersistent_reductionÚmix_order_reductionÚwant_no_x_dimÚno_x_dimr    Ústore_output_ctrÚis_native_matmulr   r[   Únative_matmulr&  rÿ   r   ÚSchedulerNoder¨   r   ÚComputedBufferÚget_reduction_typerå   Úcacher9  Úinitialize_range_treeÚrsplit_sizeÚsaved_partial_accumulate)rp   r(  r*  r›   Úoverride_persistent_reductionÚoverride_cooperative_reductionrF  rI  rm   Úvalr¨   r9  rq   s   `           €r^   rh   ÚSIMDKernel.__init__�  só  ù€ ð ÑØˆIÜ‰ÑÔØ ˆŒØ!×/Ñ/Ó1ˆŒÜ"Ó$ˆŒ	Ü+Ó-ˆÔàFLÇlÁlÄnô
ÚFT±{°vˆF”A—G‘G×$Ñ$×-Ñ-¨cÓ2Ò2Ánò
ˆŒð 79ˆÔØJLˆÔÜ(ŸšÓ0ˆÔØ (× 5Ñ 5Ó 7ˆÔð .Ñ9ñ +à×6Ñ6Ó8ð 	Ô"ð
 ?LˆÔØ-3ˆŒð -Ñ8ñ *à×5Ñ5Ó7ð 	Ô!ð
 *=ˆÔ Ø×*Ñ*Ó,ˆŒØ(,ˆŒä )§¢Ó 1ˆÔØ %ˆÔÜ�=‰=×&×&ØŸ™×3Ô3�ä˜t¤Y×%<Ñ%<×=Ó=Ü" 4§9¡9¬b×.?Ñ.?×@Ó@ØŸ	™	×4Ñ4Ó6¸%Õ?à,0�DÔ)Ùñ 4ô 
�‰ö	>ó 
ð	>ð "3ˆÔØ×"Ñ" 9Ô-àˆÔØACˆÕ%ùóa
s   Á#6Ic	                ó   • g)z¤Override template codegen. Return None to use default flow.

External template handlers (e.g. Helion) can override this method
to implement custom code generation.
NrË   )	rp   Ú
schedulingÚtemplate_nodeÚepilogue_nodesÚprologue_nodesÚbuf_name_to_prologue_groupÚprologue_preserves_zero_mask_fnÚrenderÚonly_gen_src_codes	            r^   Úcodegen_template_overrideÚ$SIMDKernel.codegen_template_overrideÒ  s   € ð  r`   c                ó   • SU S3$ )Nz<STORE_OUTPUT_Ú>rË   )rp   Úis     r^   Ú_get_store_output_subgraph_nameÚ*SIMDKernel._get_store_output_subgraph_nameä  s   € Ø ˜s !Ð$Ð$r`   c                ój   • [        U R                  5      n[        R                  " US-
  SS9U l        U$ )Nr>   )ÚstartÚstep)r²   rL  rA  rB  )rp   Útotals     r^   Úget_store_output_countÚ!SIMDKernel.get_store_output_countç  s.   € Ü�T×*Ñ*Ó+ˆÜ )§¢°e¸a±iÀaÑ HˆÔØˆr`   c                ó:   • [        S U R                   5       5      $ )Nc              3  ó8   #   • U  H  n[        U5      v •  M     g 7frX   )r6   )rü   rm   s     r^   rý   Ú0SIMDKernel.num_reduction_dims.<locals>.<genexpr>ï  s   é € ÐIº[°6Ô& v×.Ð.º[ùó   ‚)Úsumr@  rt   s    r^   Únum_reduction_dimsÚSIMDKernel.num_reduction_dimsì  s   € ô ÑI¸T¿[º[ÓIÓIÐIr`   c                ó   • [         erX   ©ÚNotImplementedError)rp   Údtypes     r^   Údtype_to_strÚSIMDKernel.dtype_to_strñ  ó   € Ü!Ð!r`   c                ó6   • U R                   R                  5       $ rX   )r*  Úselect_index_dtypert   s    r^   Úget_index_dtype_as_torch_dtypeÚ)SIMDKernel.get_index_dtype_as_torch_dtypeô  s   € Ø�}‰}×/Ñ/Ó1Ð1r`   c                ó@   • U R                  U R                  5       5      $ rX   )r{  r€  rt   s    r^   Úindex_dtypeÚSIMDKernel.index_dtype÷  s   € à× Ñ  ×!DÑ!DÓ!FÓGÐGr`   c                ó   • g©NFrË   rt   s    r^   rJ  ÚSIMDKernel.want_no_x_dimû  ó   € Ør`   c                ó  ^• [        U4S j[         5       5      nU(       + =(       d    U(       + nS	S jn/ SQn	[        [        U	5      5      n
SS/nU(       a  UnOU(       a  U
nOX«-   nU" XÆ5      nU" U	[        5      n/ n[	        U5       H|  u  nn[        U5      nUR                  U5      nUR                  U5      nUc  UOUnUR                  [        U S3TU   UUU UU=(       a    U R                  (       + UUST;   S9
5        M~     U$ )
Nc              3  ó6   >#   • U  H  oT;   d  M
  Uv •  M     g 7frX   rË   )rü   rm   r@  s     €r^   rý   Ú3SIMDKernel.construct_range_trees.<locals>.<genexpr>  s   øé € ð %
Ú!-�v¸6Ñ1A�F‰F¢ùó   ƒ	�	c                ód   ^• [        U4S jU  5       5       VVs0 s H  u  p#X2_M	     snn$ s  snnf )Nc              3  ó6   >#   • U  H  oT;   d  M
  Uv •  M     g 7frX   rË   )rü   rX  Úmasks     €r^   rý   ÚOSIMDKernel.construct_range_trees.<locals>.filtered_index_map.<locals>.<genexpr>  s   øé € Ð2UÂ#¸3ÐPTÉ·3±3Â#ùrŒ  )Ú	enumerate)Úseqr�  ÚidxrX  s    `  r^   Úfiltered_index_mapÚ<SIMDKernel.construct_range_trees.<locals>.filtered_index_map  s4   ø€ ä)2Ô2UÁ#Ó2UÔ)UôÚ)U™X˜S�’Ñ)Uòð ùó s   š,)rT   rS   rR   rU   rV   r™   rR   )r›   rœ   r�   rž   rŸ   )r…   zdict[Any, int])
r   Úall_prefixesr%  rº   r‘  r6   rÖ   rµ   r„   rH  )rp   r›   rC  ru   r@  rK  Úactive_prefixesÚno_r_dimr”  Ú	grid_dimsÚpointwise_tensor_dimsÚreduction_dimsÚtensor_dimsÚtensor_dim_mapÚgrid_dim_mapr5  rg  rm   r�   rž   r™   s       `                r^   Úconstruct_range_treesÚ SIMDKernel.construct_range_treesþ  s*  ø€ ô %ô %
Ý!-ó%
ó 
ˆð (Ô'×;¨|Ô+;ˆô	ò
 $ˆ	Ü $¤X¨iÓ%8Ó 9ÐØ ˜ˆÞØ(‰KÞØ/‰Kà/Ñ@ˆKñ ,¨KÓIˆÙ)¨)´\ÓBˆàˆÜ" ?Ö3‰IˆAˆvÜ.¨vÓ6ˆLØ'×+Ñ+¨FÓ3ˆJØ#×'Ñ'¨Ó/ˆHØ!Ñ)‘A¨xˆEØ×ÑÜ#Ø�h˜eÐ$Ø˜6‘NØØØØ'Ø(×J°×1JÑ1JÔ-JØ)Ø%Ø  F™]ñöñ 4ð& Ðr`   c                óÐ   • U R                  UU R                  U R                  R                  5       U R                  U R
                  5      nU R                  R                  U5        g rX   )rŸ  rC  r*  ru   r@  rK  r5  Úextend)rp   r›   r5  s      r^   rS  Ú SIMDKernel.initialize_range_tree5  sR   € Ø×0Ñ0ØØ×!Ñ!Ø�M‰M×&Ñ&Ó(Ø�K‰KØ�M‰Mó
ˆð 	×Ñ×Ñ Õ,r`   c                ó   • g)zZ
Hook called right before codegen with every index that will be
used in the fused kernel.
NrË   )rp   Úindicess     r^   Úfinalize_indexingÚSIMDKernel.finalize_indexing?  s   � r`   c                óp   • U R                   nSU l          U R                  XU5      X@l         $ ! X@l         f = fr†  )rC  Ústore)rp   ri   r™   r  Úpriors        r^   Ústore_reductionÚSIMDKernel.store_reductionE  s5   € Ø×%Ñ%ˆØ %ˆÔð	*Ø—:‘:˜d¨5Ó1à$)Õ!ø EÕ!ús   •- ­5c                ó   • gr†  rË   rt   s    r^   rD  Ú+SIMDKernel.should_use_cooperative_reductionM  rˆ  r`   c                ó   • gr†  rË   rt   s    r^   rG  Ú*SIMDKernel.should_use_persistent_reductionP  rˆ  r`   c                ót   • [        [        R                  R                  S U R                   5       5      5      $ )Nc              3  óT   #   • U  H  oR                   R                  5       v •  M      g 7frX   )rk   r{   ©rü   r8  s     r^   rý   Ú(SIMDKernel.var_ranges.<locals>.<genexpr>U  s"   é € ð *Ú4D¨D—‘×%Ñ%×'Ð'Ò4Dùs   ‚&()r'  rA  ÚchainÚfrom_iterabler5  rt   s    r^   rk   ÚSIMDKernel.var_rangesS  s4   € ÜÜ�O‰O×)Ñ)ñ *Ø48×4DÒ4Dó*ó ó
ð 	
r`   c                ó:   • [        S U R                   5       5      $ )Nc              3  óP   #   • U  H  n[        UR                  S L5      v •  M     g 7frX   )rÜ   r�   r³  s     r^   rý   Ú0SIMDKernel.triton_tensor_ndim.<locals>.<genexpr>[  s#   é € ÐQÒ@P¸”3�t—‘¨dÐ2×3Ð3Ò@Pùs   ‚$&)rt  r5  rt   s    r^   Útriton_tensor_ndimÚSIMDKernel.triton_tensor_ndimZ  s   € ÜÑQÀ×@PÒ@PÓQÓQÐQr`   c                ó\   • S/U R                  5       -  nSX!'   SSR                  U5       S3$ )Nr†   Ú:Ú[r¢   Ú])r»  Újoin)rp   rg  rÒ   s      r^   Úindexing_size_strÚSIMDKernel.indexing_size_str]  s7   € Ø�˜4×2Ñ2Ó4Ñ4ˆØˆ‰Ø�4—9‘9˜UÓ#Ð$ AÐ&Ð&r`   c                ó  • S/U R                  5       -  nU R                   H_  nUR                  c  M  UR                  (       a  U R                  (       d  M6  UR
                  R                  5        S3XR                  '   Ma     U$ )NÚ1ÚBLOCK)r»  r5  r�   ru   rC  rm   Úupper)rp   rÒ   r8  s      r^   Údense_size_listÚSIMDKernel.dense_size_listb  sp   € Ø�˜×/Ñ/Ó1Ñ1ˆØ×$Ô$ˆDØ�‰Ñ&Ùà×$×$¨×(=×(=Ñ(=Ø,0¯K©K×,=Ñ,=Ó,?Ð+@ÀÐ)F�—o‘oÓ&ñ %ð ˆr`   c                ó   • UR                   nUR                  c  U R                  5       nU SU S3$ S/U R                  5       -  nSXAR                  '   SR	                  U5      nU SUR                  5        SU S3nU$ )	Nzmask = tl.full(z, True, tl.int1)r†   r¾  r¢   zmask = tl.full([zBLOCK], True, tl.int1)[rÀ  )rm   r�   Údense_size_strr»  rÁ  rÇ  )rp   ÚentryrT   ÚsizestrrÒ   ÚsuffixÚouts          r^   Úcreate_constant_maskÚSIMDKernel.create_constant_maskl  s�   € Ø�L‰LˆØ×ÑÑ#Ø×)Ñ)Ó+ˆGØ�S˜¨ yÐ0@ÐAÐAØ�˜4×2Ñ2Ó4Ñ4ˆØ"%ˆ×ÑÑØ—‘˜5Ó!ˆØ�Ð# A§G¡G£I ;Ð.EÀfÀXÈQÐOˆØˆ
r`   c                óL   • U R                  5       nSSR                  U5       S3$ )Nr¿  r¢   rÀ  )rÈ  rÁ  ©rp   rÒ   s     r^   rË  ÚSIMDKernel.dense_size_strw  s)   € Ø×$Ñ$Ó&ˆØ�4—9‘9˜UÓ#Ð$ AÐ&Ð&r`   c                ó  • [        U[        5      (       d  U$ UR                  S   nU R                  R	                  U5      =nc  U$ [        XUR                  05      n[        R                  R                  R                  U5      n[        UUR                  R                  5       UR                  R                  [        R                  R                   UR                  R"                  5      R%                  5       05      $ ©Nr   )rÿ   r   r  r´   rÖ   r9   r¶   r=   r®   r¯   r7  ro   r«   r·   r�   r�   r‘   rl   rx   )rp   r™   rT   Ú	tree_nodeÚ	new_indexs        r^   r7  Ú)SIMDKernel.combine_modular_indexing_pairs{  sÃ   € Ü˜%¤×1Ñ1ØˆLØ�J‰J�q‰MˆØ×.Ñ.×2Ñ2°1Ó5Ð5ˆIÑ>ØˆLÜ˜u¨)¯.©.Ð&9Ó:ˆ	Ü—G‘G×$Ñ$×CÑCÀIÓNˆ	äØà—‘×(Ñ(Ó*¨I¯N©N×,AÑ,AÜ—G‘G—K‘K §¡×!5Ñ!5ó-ç‘&“(ðó
ð 	
r`   c                óÂ   • [         R                  R                  R                  U5      =n(       a  Uu  pE[	        U R                  XB5      U5      $ U R                  X5      $ rX   )r=   r®   r¯   Úexpand_floor_divr   Ú_combine_contiguous_dims)rp   r™   r8  Ú
expand_resrØ  Údenominators         r^   r6  Ú"SIMDKernel.combine_contiguous_dims�  sU   € ô Ÿ™×)Ñ)×:Ñ:¸5ÓAÐAˆ:ÕAØ%/Ñ"ˆIÜ˜D×9Ñ9¸)ÓJÈKÓXÐXà×0Ñ0°Ó=Ð=r`   c                ó˜  • [        U[        R                  [        R                  45      (       a  U$ UR	                  U5      u  p4[        U5      S::  a  U$ [        R                  R                  R                  X4[        U/X45      5      u  pVnXT:X  a  U$ UR                  U5      n[        U[        [        X6" U5      5      5      5      n	U	$ )z9
More aggressive simplification to merge contiguous dims
r>   )rÿ   r�   r  r   rÚ   r  r=   r®   r¯   Ú_simplify_loopsrA   rÁ   r9   r'  Úzip)
rp   r™   r8  rÑ   rÒ   Ú	new_sizesÚreindexÚ_pruneÚnew_index_varsrØ  s
             r^   rÜ  Ú#SIMDKernel._combine_contiguous_dims–  s²   € ô �eœeŸm™m¬U¯\©\Ð:×;Ñ;ØˆLØ ×/Ñ/°Ó6Ñˆ
Üˆu‹:˜‹?ØˆLÜ%&§W¡W×%5Ñ%5×%EÑ%EØÔ7¸¸ÀÓSó&
Ñ"ˆ	˜Fð ÓØˆLØŸ™¨	Ó2ˆÜ˜u¤d¬3¨z¸7À>Ó;RÓ+SÓ&TÓUˆ	ØÐr`   c                ó    ^ ^• T R                   S   R                  =(       d    T R                  m[        R                  U U4S j5       nU" 5       $ )Néÿÿÿÿc               3  ó  >#   • T R                   R                  5       (       d  T R                  (       a   eS v •  g T(       a  T R                  5         ST l         S v •  T(       a  T R                  5         ST l        g ! ST l        f = f7f)NFT)r*  ru   rC  Úcodegen_body)rp   Úshould_flushs   €€r^   ÚctxÚ)SIMDKernel.disable_reduction.<locals>.ctx­  sn   øé € à—=‘=×-Ñ-×/Ñ/Ø×0×0Ð0Ð0ÛØÞð ×!Ñ!Ô#Ø$)ˆDÔ!ð-ÛÞà×%Ñ%Ô'à(,�Õ%ø¨�Õ%üs   ƒAB	ÁA= Á5B	Á=	BÂB	)r5  rœ   rE  Ú
contextlibÚcontextmanager)rp   rí  rì  s   ` @r^   Údisable_reductionÚSIMDKernel.disable_reductionª  sE   ù€ Ø×'Ñ'¨Ñ+×3Ñ3×Q°t×7QÑ7Qˆä	×	"Ñ	"õ	-ó 
#ð	-ñ$ ‹uˆr`   c                óÈ   • [        U5      [        U R                  5      :X  d   e[        XR                  5       VVs/ s H  u  p#UR                  U5      PM     snn$ s  snnf rX   )r  r5  râ  rÁ   )rp   r»   re   Úrangess       r^   Ú
set_rangesÚSIMDKernel.set_rangesÂ  s^   € Ü�7‹|œs 4×#3Ñ#3Ó4Ó4Ð4Ð4ô #& g×/?Ñ/?Ô"@ô
â"@‘�ð ×Ñ˜VÖ$Ù"@ò
ð 	
ùó 
s   ½Ac                óÂ  ^^^^• [        S U 5       5      (       a  U  Vs/ s H  n/ PM     sn/ 4$ [        R                  R                  mU  Vs/ s H  n/ PM     snmU  Vs/ s H  nTR	                  U5      PM     snm[
        R                  " 5       mSUUUU4S jjn      SS jn/ nSnU GHt  n	/ n
U	 GHV  nTR                  US5      (       a  U
R                  S 5        M/  U[        T5      :  aJ  TR                  TU   S5      (       a0  US-  nU[        T5      :  a  TR                  TU   S5      (       a  M0  [        T5      S:H  =(       a    TS   S:H  nUS	-   [        T5      :  a§  TR                  UTU   TUS-      -  5      (       a„  U(       a}  TR                  UTU   TUS-      -  5      (       d  [        eTU   nTUS-      n[        X½U-  5      nU
R                  U" Xï/U" X�5      U" US-   U5      U" US	-   U5      /5      5        GM^  US-   [        T5      :  a©  TR                  UTU   5      (       d$  TR                  [        UTU   5      S5      (       ak  TR                  UTU   5      (       d  [        UTU   5      eTU   n[        UTU   5      nU
R                  U" U/U" X�5      U" US-   U5      /5      5        GM  U[        T5      :  d  GM+  U
R                  [        R                  " U" X‹5      5      5        GMY     UR                  U
5        GMw     [        S
 T 5       5      (       d   ST SU 35       eTU4$ s  snf s  snf s  snf )Nc              3  ó>   #   • U  H  n[        U5      S :H  v •  M     g7f©r   N©r  )rü   re   s     r^   rý   Ú5SIMDKernel._split_iteration_ranges.<locals>.<genexpr>Ð  s   é € Ð6ªg FŒs�6‹{˜aÖªgùs   ‚c                óÚ   >• TR                  U5      nTR                  TU    U5      (       d  [        TU    U5      e[        TU    U5      TU '   TU    R	                  U5        [        T5      $ rX   )r?  Ústatically_known_multiple_ofÚ	CantSplitr   rµ   r²   )rg  r¶   Ú
new_rangesÚ	remainingÚsvÚ	var_counts     €€€€r^   Ú	add_rangeÚ5SIMDKernel._split_iteration_ranges.<locals>.add_rangeØ  si   ø€ Ø—;‘;˜tÓ$ˆDØ×2Ñ2°9¸Q±<À×FÑFÜ 	¨!¡¨dÓ3Ð3ä# I¨a¡L°$Ó7ˆI�a‰LØ�q‰M× Ñ  Ô&Ü˜	“?Ð"r`   c                óV   ^ ^• [        T5      [        T 5      S-   :X  d   eSUU 4S jjnU$ )z`
Builds the nested expression:
  ((...((s1*v[i1] + v[i2]) * s2 + v[i3]) ... ) * sk + v[i(k+1)])
r>   c                óZ   >• U TS      n[        TTSS  5       H  u  p#X!-  X   -   nM     U$ )Nr   r>   )râ  )Ú	flat_varsr¶   rØ   r“  ÚidxsrÒ   s       €€r^   ÚgetterÚISIMDKernel._split_iteration_ranges.<locals>.make_combined.<locals>.getterê  s=   ø€ Ø   a¡Ñ)�Ü! %¨¨a¨b¨Ö2‘F�AØ™8 i¡nÑ4’Dñ 3à�r`   )r  rá   r…   r‚   rú  )rÒ   r  r	  s   `` r^   Úmake_combinedÚ9SIMDKernel._split_iteration_ranges.<locals>.make_combinedá  s0   ù€ ô �t“9¤ E£
¨Q¡Ó.Ð.Ð.÷ð ð ˆMr`   r   r>   c                ó6   • [         R                  R                  $ rX   )r�   r�   ÚZero)Ú_s    r^   rÌ   Ú4SIMDKernel._split_iteration_ranges.<locals>.<lambda>ø  s   € ´E·G±G·L²Lr`   é   ré  r   c              3  óz   #   • U  H1  n[         R                  R                  R                  U5      S :H  v •  M3     g7f)r>   N)r=   r®   r¯   Ú	size_hintrû   s     r^   rý   rû  V  s*   é € ÐIºy¸!”1—7‘7×#Ñ#×-Ñ-¨aÓ0°AÖ5ºyùs   ‚9;zfailed to set ranges Ú )rg  rÜ   r¶   r‚   r…   rÜ   )rÒ   rá   r  z	list[int]r…   z(Callable[[list[sympy.Expr]], sympy.Expr])r  r=   r®   r¯   r?  rA  rB  r°   rµ   r  Ústatically_known_gtrý  rþ  r   ÚoperatorÚ
itemgetter)Úgroupsr»   Úgroupr  Úgr  r  Úreturn_getters_groupsÚcurrent_groupÚlength_groupÚreturn_gettersÚsizeÚis_bmm_then_pwÚsize1Úsize2Úsize3rÿ  r   r  r  s                   @@@@r^   Ú_split_iteration_rangesÚ"SIMDKernel._split_iteration_rangesÉ  s³  û€ ô Ñ6©gÓ6×6Ñ6Ù$*Ó+¢F˜5“B¡FÑ+¨RÐ/Ð/ä�W‰W×ÑˆÙ:@Ó-Aº&°Q«b¹&Ñ-Aˆ
Ù-3Ó4ªV¨�R—[‘[ –^©VÑ4ˆ	Ü—O’OÓ%ˆ	÷	#ò 	#ð	Ø#ð	Ø+4ð	à5ô	ð" !#ÐØˆÜ#ˆLØˆNÜ$�Ø×-Ñ-¨d°A×6Ñ6Ø"×)Ñ)Ñ*@ÔAÙà#¤c¨)£nÓ4¸×9SÑ9SØ˜mÑ,Ø÷:ñ :ð
 " QÑ&�Mð $¤c¨)£nÓ4¸×9SÑ9SØ˜mÑ,Ø÷:ó :ô$ "% Y£°1Ñ!4×!K¸À2¹È!Ñ9K�à! AÑ%¬¨I«Ó6Ø×.Ñ.Ø˜i¨Ñ6¸À=ÐSTÑCTÑ9UÑU÷ñ ö 'ð ×:Ñ:Ø˜i¨Ñ6¸À=ÐSTÑCTÑ9UÑU÷ñ ô (˜à% mÑ4�EØ% m°aÑ&7Ñ8�EÜ$ T°5©=Ó9�EØ"×)Ñ)á%Ø"˜Ná )¨-Ó ?Ù )¨-¸!Ñ*;¸UÓ CÙ )¨-¸!Ñ*;¸UÓ Cðó÷
ð # QÑ&¬¨Y«Ó7Ø×*Ñ*¨4°¸=Ñ1I×JÑJð ×*Ñ*¬8°D¸)ÀMÑ:RÓ+SÐUV×WÑWð ×:Ñ:Ø˜i¨Ñ6÷ñ ô (¨¨i¸Ñ.FÓGÐGà% mÑ4�EÜ$ T¨9°]Ñ+CÓD�EØ"×)Ñ)á%Ø"˜Gá )¨-Ó ?Ù )¨-¸!Ñ*;¸UÓ Cðó÷	ð %¤s¨9£~Ö5Ø&×-Ñ-ä$×/Ò/±	¸-Ó0NÓO÷ñu %ð| "×(Ñ(¨×8ñA $ôD ÑI¹yÓI×IÑIð 	
Ø# I ;¨a°¨yÐ9ó	
ÐIð Ð0Ð0Ð0ùòS ,ùò .BùÚ4s    MÁMÁ"Mc                ó*  • [         R                  R                  n[        US   5      S:X  af  UR	                  U[
        R                  R                  5      (       d7  UR	                  [        U5      [        US   5      U-  5      (       a  US   U/4$ U$ )z1Fill in the reduction numel of lengths if missingr>   r   )	r=   r®   r¯   r  r°   r�   r�   r‘   r8   )Úclsr  r»   Úreduction_numelr¯   s        r^   Úprepare_split_iteration_lengthsÚ*SIMDKernel.prepare_split_iteration_lengths\  sƒ   € ô —7‘7×#Ñ#ˆÜˆw�q‰z‹?˜aÓØ×0Ñ0°Ä%Ç'Á'Ç+Á+×NÑNØ×0Ñ0Ü˜fÓ%Ü˜g a™jÓ)¨OÑ;÷ñ ð
 ˜A‘J Ð 1Ð2Ð2àˆr`   c                ól   • U R                  XU5      n U R                  X5        g! [         a     gf = f©NTF)r)  r$  rþ  )r'  r  r»   r(  s       r^   Úis_compatibleÚSIMDKernel.is_compatiblep  s>   € ð ×5Ñ5°fÀÓWˆð	Ø×'Ñ'¨Ô8ØøÜó 	Ùð	ús   ”& ¦
3²3c                óX  • U R                    Vs0 s H  o"R                  UR                  _M     nnU R                  (       d7  U H1  n[	        U5      (       d  M  [
        R                  R                  X4'   M3     / UR                  5       QnU R                  XQU R                  5      $ s  snf )aå  
Split and set iteration ranges for the kernel based on the provided lengths.

This method maps the kernel's tiling structure to the node's iteration space,
handling both pointwise and reduction dimensions appropriately.

Args:
    lengths: A sequence of sequences of symbolic expressions representing
            the sizes of different dimensions for each node.

Returns:
    A list of lists of symbolic expressions representing the mapped
    iteration variables for each dimension.
)r5  rm   rl   rC  r6   r�   r�   r‘   r¦   Úmap_kernel_groups_to_node_sizesrõ  )rp   r»   Úrtr(  rm   r  s         r^   Úsplit_and_set_rangesÚSIMDKernel.split_and_set_ranges  s‹   € ð$ 15×0@Ò0@ÓAÒ0@¨"—)‘)˜RŸX™XÒ%Ñ0@ˆÐAð ×$×$Û �Ü& v×.Ó.Ü%*§W¡W§[¡[�F“Nñ !ð
 $�6—=‘=“?Ð#ˆð ×3Ñ3°FÀTÇ_Á_ÓUÐUùò Bs   � B'c           
     óT  • [        U5      [        U5      :X  a%  [        S [        X!5       5       5      (       a  U" U6 $ U R                  X5      u  pE/ [        R
                  R                  U" U6 5      QnU VVs/ s H  ow Vs/ s H
  oˆ" U5      PM     snPM     snn$ s  snf s  snnf )aY  
We may want to fuse `for i0 in s0*s1` into a tiled kernel with groups (s0, s1).

To do this we need to split up the iteration space of i0 into something like:
    for i1 in s0:
      for i2 in s1:
        i0 = i1*s1 + i2
        ....

This function matches and resplits lengths to the groups of
this kernel to enable tiled + non-tiled fusions.
c              3  ó–   #   • U  H?  u  p[         R                  R                  R                  [	        U5      U-
  5      S :H  v •  MA     g7frù  ©r=   r®   r¯   r?  r8   )rü   rT   r  s      r^   rý   Ú=SIMDKernel.map_kernel_groups_to_node_sizes.<locals>.<genexpr>´  s=   é € ð /
â,‘�ô �G‰G×Ñ×%Ñ%¤m°AÓ&6¸Ñ&:Ó;¸qÖ@Ú,ùs   ‚AA	)r  r  râ  r$  rA  rµ  r¶  )	r'  r  r»   rõ  rÿ  r  r¼   ÚfnsÚfns	            r^   r0  Ú*SIMDKernel.map_kernel_groups_to_node_sizes¡  s    € ô& ˆw‹<œ3˜v›;Ó&¬3ñ /
ä˜GÔ,ó/
÷ ,
ñ ,
ñ ˜wÐ'Ð'à,/×,GÑ,GÈÓ,XÑ)ˆ
ØL”Y—_‘_×2Ñ2±:¸zÐ3JÓKÐLˆÙ8MÔNÒ8M°¨Ó,ª "��H–©Ô,Ñ8MÒNÐNùÒ,ùÓNs   Á:	B$ÂBÂB$ÂB$c                ó6   • [        U[        R                  5      $ rX   )r   r   ÚTMP©rp   r™   s     r^   Úis_indirect_indexingÚSIMDKernel.is_indirect_indexing¾  s   € ä" 5¬$¯(©(Ó3Ð3r`   c                ó  ^• U R                  U5      (       a  gS/[        U R                  5      -  nUR                   Hn  nX0R                  ;  a  M  U R                  U   n[        UR                  [        5      (       d   eX$R                  R                  ==   UR                  -  ss'   Mp     [        R                  R                  R                  m[        U4S j[        X R                  R!                  5       5       5       5      $ )NFr>   c              3  óJ   >#   • U  H  u  pT" U5      T" U5      :g  v •  M     g 7frX   rË   )rü   Ú	idx_rangeÚ
iter_ranger?  s      €r^   rý   Ú,SIMDKernel.is_broadcasted.<locals>.<genexpr>Ó  s*   øé € ð 
â)PÑ%�	ñ �YÓ¡8¨JÓ#7Ö7Ú)Pùs   ƒ #)r>  r  r@  rÕ   r´   rÿ   rä   r„   r™   re   r=   r®   r¯   r?  Úanyrâ  r¦   )rp   r™   Úindex_numelsrx   rÌ  r?  s        @r^   Úis_broadcastedÚSIMDKernel.is_broadcastedÂ  sÒ   ø€ à×$Ñ$ U×+Ñ+Øà�sœS §¡Ó-Ñ-ˆØ×(Ô(ˆFØ×2Ñ2Ó2áØ×)Ñ)¨&Ñ1ˆEÜ˜eŸl™lÔ,?×@Ñ@Ð@Ð@ØŸ™×+Ñ+Ó,°·±Ñ<Õ,ñ )ô —7‘7×#Ñ#×,Ñ,ˆÜô 
ä),¨\¿;¹;×;MÑ;MÓ;OÔ)Pó
ó 
ð 	
r`   c                ó¾   • [        U[        5      (       a)  SSR                  [        U R                  U5      5       S3$ U R                  U R                  U5      5      $ )a`  
Convert an index expr to a string that can be used in output code.
e.g. a sympy expression "s2" may actually appear as "ks1" in the generated kernel.

Index expressions often need to be passed in as arguments to the triton kernel.
Rename_indexing and codegen_indexing keep track of the needed indices and add
new parameters to the function signature.
r¿  r¢   rÀ  )rÿ   r%  rÁ  ÚmapÚindex_to_strr.  Úrename_indexingr=  s     r^   rK  ÚSIMDKernel.index_to_strØ  sQ   € ô �eœT×"Ñ"Ø�t—y‘y¤ T×%6Ñ%6¸Ó!>Ó?Ð@ÀÐBÐBØ�z‰z˜$×.Ñ.¨uÓ5Ó6Ð6r`   c                ó¨  • U R                  U5      n[        U[        R                  R                  R
                  5      n[        UR                  [        R                  5      5      (       d-  [        UR                  [        R                  5      5      (       a3  UR                  [        R                  R                  R
                  5      n[        UR                  [        R                  5      5      (       a’  UR                  [        R                  5       Ho  nUR                  n[        U5      S:”  d  M   [        S U 5       5      (       d  M9  U[        R                  R                  R                  U5      0n[        X5      nMq     U R                  U5      n[        U[         5      (       d  UOUR"                  S   nU R%                  U5      $ )Nr   c              3  óv   #   • U  H/  n[        U[        R                  [        R                  45      v •  M1     g 7frX   )r   r   rú   ÚPRECOMPUTED_SIZErû   s     r^   rý   Ú.SIMDKernel.prepare_indexing.<locals>.<genexpr>ù  s0   é € ð ,â$˜ô # 1¤t§y¡y´$×2GÑ2GÐ&H×IÐIÚ$ùs   ‚79)r9  r9   r=   r®   r¯   Úprecomputed_replacementsr  Úatomsr�   ÚfloorÚceilingÚsubsrÕ   r  Úlookup_precomputed_sizerÿ   r   r  Úcodegen_indexing)rp   r™   Úar  ÚreplacementsÚ
simp_indexs         r^   Úprepare_indexingÚSIMDKernel.prepare_indexingå  sQ  € ð ×&Ñ& uÓ-ˆÜ˜5¤!§'¡'×"2Ñ"2×"KÑ"KÓLˆäˆu�{‰{œ5Ÿ;™;Ó'×(Ñ(¬C°·±¼E¿M¹MÓ0J×,KÑ,KØ—J‘JœqŸw™w×/Ñ/×HÑHÓIˆEô ˆu�{‰{œ5Ÿ=™=Ó)×*Ñ*Ø—[‘[¤§¡Ö/�ð Ÿ.™.�Ü�w“< !Õ#¬ñ ,á$ó,÷ )ó )ð %&¤q§w¡w×'7Ñ'7×'OÑ'OÐPQÓ'RÐ#S�LÜ& uÓ;’Eñ 0ð ×+Ñ+¨EÓ2ˆ
ô )¨´X×>Ñ>‰JÀJÇOÁOÐTUÑDVð 	ð ×$Ñ$ ZÓ0Ð0r`   c                óŒ   • U R                    Vs/ s H(  oR                  (       a  U R                  (       d  M&  UPM*     sn$ s  snf rX   )r5  ru   rC  )rp   Úts     r^   Úactive_range_treesÚSIMDKernel.active_range_trees  s6   € à×'Ò'ó
Ú'�!¯~¯~À×AVÕAV�AÑ'ñ
ð 	
ùò 
s
   �%A¸Ac                ó8  • [         R                  R                  R                  XR	                  5       5      n[        UR                  [        S9 HÆ  nX R                  ;   d  M  0 nU R                  U   R                  5        H.  n[         R                  R                  R                  U5      X4'   M0     [        U5      S:”  a5  [        U R                  U   R                  U5      U R                  U   l        U R                  U   R                  5         MÈ     U$ )NrÎ   r   )r=   r®   r¯   r4  rk   ÚsortedrÕ   r   r´   r  rW  r  r9   r¶   rè   )rp   r¶   ÚsymrZ  Úpss        r^   rX  ÚSIMDKernel.codegen_indexing  sæ   € Ü�w‰w×Ñ×4Ñ4°T¿?¹?Ó;LÓMˆÜ˜$×+Ñ+´Ô5ˆCØ×+Ñ+Õ+ð  "�Ø×/Ñ/°Ñ4×EÑEÖG�BÜ'(§w¡w×'7Ñ'7×'OÑ'OÐPRÓ'S�LÓ$ñ Hä�|Ó$ qÓ(Ü6@Ø×-Ñ-¨cÑ2×7Ñ7Ø$ó7�D×)Ñ)¨#Ñ.Ô3ð ×%Ñ% cÑ*×2Ñ2Ö4ñ 6ð ˆr`   c                ó   • [        S5      e)NzNYI: codegen_nan_checkrx  rt   s    r^   Úcodegen_nan_checkÚSIMDKernel.codegen_nan_check!  s   € Ü!Ð":Ó;Ð;r`   c                ó¦   • [         R                  R                  n[        U R                  R
                  5       H  nUR                  U5        M     g rX   )r=   r®   Úwrapper_coderº   r  Úworkspace_argsÚgenerate_workspace_deallocation)rp   ÚwrapperÚwss      r^   Údeallocate_workspacesÚ SIMDKernel.deallocate_workspaces$  s8   € Ü—'‘'×&Ñ&ˆÜ˜4Ÿ9™9×3Ñ3Ö4ˆBØ×3Ñ3°BÖ7ò 5r`   c                ó   • [        S5      e)NzNYI: call_kernelrx  )rp   ri   r¨   Údeallocate_wss       r^   Úcall_kernelÚSIMDKernel.call_kernel)  s   € ô "Ð"4Ó5Ð5r`   c              #  óþ   #   • U R                   nU R                  nU(       a  [        R                  " X5      n[        R
                  " U5      nXl         X l         Uv •  X0l         X@l        g! X0l         X@l        f = f7f)z:Context manager to add an additional mask to tl.load/storeN)Ú
_load_maskÚ_load_otherr;   Úlogical_andr<   Ú_unwrap)rp   r�  r  rª  Ú	prior_vals        r^   Ú
mask_loadsÚSIMDKernel.mask_loads.  sj   é € ð
 —‘ˆØ×$Ñ$ˆ	ÞÜ—?’? 4Ó/ˆDä×!Ò! $Ó'ˆØŒØ Ôð	)àŠJà#ŒOØ(Õøð $ŒOØ(Õüs   ‚AA=ÁA, ÁA=Á,A:Á:A=c                ó&  • U R                   R                  5        VVs0 s H  u  p#X#R                  _M     nnn[        X5      n0 nU R                   H5  n[        UR                  5      n[        XXS05      [        XXS05      -
  Xh'   M7     U$ s  snnf )a  
This gets the stride of the index for each of the tiling variables
(technically, it does it at index 0)

For example, if
xindex = x0 + 512*x1 + 1024*r0
x0 = (xindex//512)
x1 = (xindex % 512)
r0 = rindex // 1024

this function would return
{xindex: 512, rindex: 1024}
r>   r   )r´   r{   r¶   r9   r5  r7   ri   )	rp   r™   ÚkÚvÚindex_to_tile_indexesÚindex_in_tile_varsÚstridesÚ
range_treerØ   s	            r^   Úget_strides_of_loadÚSIMDKernel.get_strides_of_loadB  s’   € ð 8<×7LÑ7L×7RÑ7RÔ7TÔ UÒ7T©t¨q §F¡F¢Ñ7TÐÑ UÜ'¨ÓEÐØˆØ×*Ô*ˆJÜ" :§?¡?Ó3ˆAÜ#Ð$6¸A¸Ó?Ä*Ø"¨ FóCñ ˆG‹Jñ +ð
 ˆùó !Vs   žBc                ód   • [        U[        5      (       a  [        [        X5      5      $ U " U5      $ rX   )rÿ   ÚtuplerJ  )r9  r  s     r^   Ú_map_tuple_or_scalarÚSIMDKernel._map_tuple_or_scalarZ  s(   € ä�eœU×#Ñ#Üœ˜R›Ó(Ð(Ù�%‹yÐr`   c                óÆ   • [         R                  " U R                  R                  5       Vs/ s H  nUR	                  5       PM     nn[        [        S U5      5      $ s  snf rX   )rI   Ú
only_nodesr*  r&  Úestimate_flopsrt  Úfilter)rp   r¨   Úflopss      r^   r�  ÚSIMDKernel.estimate_flops`  s[   € ô +×5Ò5°d·m±m×6QÑ6QÔRó
âR�ð ×ÑÖ!ÙRð 	ð 
ô ”6˜$ Ó&Ó'Ð'ùò	
s   ­Ac           	     ó"  • / n[        [        U R                  R                  R	                  5       5      5      nU R                  R                  5       u  p4  nU R                  R                  5       n[        R                  R                  R                  [        U R                  R	                  5       5      5      n[        U5       GH;  u  pxX…;  a  UR                  S5        M  [        R                  R!                  U5      n	[        R                  R                  R                  U	5      n
X¦:”  a  ["        [$           " 5       nSnXX    HT  n['        U[(        [*        45      (       a  UR-                  SU 35        US-  nM9  UR-                  UR.                  5        MV     [        U5      U-  nOU
n[        R                  R1                  U5      n[3        U5      nUR                  UU-  S[5        Xr:  5      -   -  5        GM>     [7        U5      $ )aË  
Try the best to estimate the total size (in bytes) of the
kernel's inputs and outputs, which is used for estimating the memory
throughput of this kernel. This information is used for checking how
far we are from the peak memory bandwidth. It's important that
we want to avoid overestimating the sizes of the inputs and outputs,
because it can wrongfully give us a very large memory traffic value,
which may be even larger than the theoretical bandwidth and thus
become very misleading. This is particularly problematic for cases
where we slice some inputs. In those cases, we should only count
the size of the "slices" instead of the original inputs, because
only the slices contribute to the real memory traffic.
r   Úno_index_dep_r>   )r  r:   r  Úinplace_buffersr¦   Úpython_argdefsr*  Úbuf_accessesr=   r®   r¯   rÅ   r8   r@  r‘  rµ   Ú	get_numelr   r   rÿ   r#   r$   rÓ   r™   Ú	get_dtyper3   rÜ   rt  )rp   ÚnbytesÚninplace_argsr  Ú	call_argsr•  Ú	out_numelrg  r  Ú	arg_numelÚbuf_sizer¥  Úno_index_dep_countÚdeprl   rz  Ú
dtype_sizes                    r^   Úestimate_kernel_num_bytesÚ$SIMDKernel.estimate_kernel_num_bytesg  sª  € ð ˆÜœF 4§9¡9×#<Ñ#<×#CÑ#CÓ#EÓFÓGˆØ!ŸY™Y×5Ñ5Ó7Ñˆ�a˜Ø—}‘}×1Ñ1Ó3ˆô —G‘G×$Ñ$×6Ñ6Ü˜$Ÿ+™+×,Ñ,Ó.Ó/ó
ˆ	ô   	×*‰FˆAð Ó&Ø—‘˜aÔ ÙÜŸ™×)Ñ)¨#Ó.ˆIÜ—w‘w×'Ñ'×9Ñ9¸)ÓDˆHØÓ#ô %¤Sš/Ó+�Ø%&Ð"Ø'Ô,�CÜ! #¬´Ð'9×:Ñ:ØŸ™ mÐ4FÐ3GÐ$HÔIØ*¨aÑ/Ò*àŸ™ C§I¡IÖ.ñ -ô ˜G› yÑ0‘à �Ü—G‘G×%Ñ% cÓ*ˆEÜ'¨Ó.ˆJà�M‰M˜% *Ñ,°´C¸Ñ8IÓ4JÑ0JÑK×Lñ; +ô< �6‹{Ðr`   c           	     ó‚  • [        U R                  R                  5      S:X  aG  [        U R                  R                  5      S:X  a$  [        U R                  R                  5      S:X  a  gU R                  R                  5       u  p#pESnU GHr  n[        R                  R                  U5      nU(       d  M,  UR                  5       n	[        U	R                  5      S:X  d  MW  [        U	R                   V
s/ s H  oªS:X  d  M
  U
PM     sn
5      S:X  a  MŠ  [        R                  " U	R                  5      nUc  UnM±  Xk:w  d  M¸  [        SU S3SU S	U 3-   5      n[        R!                  U5        U Vs/ s Ht  n[        R                  R                  U5      (       aK  [        R                  " [        R                  R#                  U5      R                  5       R                  5      OSPMv     nnU Vs/ s H`  n[        R                  R                  U5      (       a7  [        R                  R#                  U5      R                  5       R                  OSPMb     nnU Vs/ s HE  nU[        R                  R$                  ;   a  S
O!U[        R                  R&                  ;   a  SOSPMG     nnU V
s/ s H  oªR(                  PM     nn
[        SU SU SU 3SU SU S3-   5      n[        R!                  U5          g   [+        SU S35      n[        R!                  U5        gs  sn
f s  snf s  snf s  snf s  sn
f )zZ
Print message if the kernel have mixed layout inputs.
Only care about 4D tensor for now.
r>   r   Nr  r   zExpected stride order z, but found stride orderr  z for kernel Ú
GraphInputÚIntermediateBufferz  param names z
  buf names z
  strides z	
  sizes z
  sources Ú
z%All the inputs for the triton kernel z have uniform layout)r  r  Úinput_buffersÚoutput_buffersr“  r”  r=   r®   Útry_get_bufferÚ
get_layoutr  r   Úget_stride_orderÚstrider-   ÚlogÚwarningÚ
get_bufferÚgraph_inputsÚname_to_bufferri   r+   )rp   r0  Úargdefsrš  Ú
_signaturer  Úuniform_stride_orderÚarg_nameÚbufÚlayoutrT   Ústride_orderÚmsgri   Ústride_order_listÚ	size_listÚsource_listÚargdef_namess                     r^   Úwarn_mix_layoutÚSIMDKernel.warn_mix_layout©  s  € ô �—	‘	×'Ñ'Ó(¨AÓ-Ü�D—I‘I×,Ñ,Ó-°Ó2Ü�D—I‘I×-Ñ-Ó.°!Ó3ð
 à,0¯I©I×,DÑ,DÓ,FÑ)ˆ˜JØ#Ðä!ˆHÜ—'‘'×(Ñ(¨Ó2ˆCÞÙØ—^‘^Ó%ˆFÜ�6—;‘;Ó 1Õ$ä 6§;¢;Ó9¢;˜a°q±&Ÿ¡;Ñ9Ó:¸aÓ?ÙÜ!×2Ò2°6·=±=ÓA�Ø'Ñ/Ø+7Ò(Ø)Õ9Ü%Ø0Ð1EÐ0FÐF^Ð_Ø˜l˜^¨<¸°}ÐEñFó�Cô —K‘K Ô$ñ %.ó)ò %.˜Dô Ÿ7™7×1Ñ1°$×7Ñ7ô ×+Ò+ÜŸG™G×.Ñ.¨tÓ4×?Ñ?ÓA×HÑHôð "ò	"ñ
 %.ð &ð )ñ %.ó	!ò %.˜Dô Ÿ7™7×1Ñ1°$×7Ñ7ô Ÿ™×*Ñ*¨4Ó0×;Ñ;Ó=×BÒBà!ò"ñ %.ð	 ð !ñ %.ó#ò %.˜Dð	  ¤1§7¡7×#7Ñ#7Ó7ñ %ð  ¤1§7¡7×#9Ñ#9Ó9ñ 2à!ò	"ñ
 %.ð  ð #ñ 5<Ó#<²G¨q§F¤F±G�LÐ#<Ü%Ø(¨¨°nÀYÀKÈ|Ð\mÐ[nÐoØ& y k°¸k¸]È"ÐMñNó�Cô —K‘K Ô$Ùña "ôb Ø3°K°=Ð@TÐUó
ˆô 	�‰�CÕùò[ :ùò)ùò!ùò#ùò $=s'   Ã6	L(
ÄL(
Å5A;L-Ç6A'L2É#AL7Ê5L<c                óˆ  • [         R                  " XSU5      nSU l        [         R                  " U R                  R
                  U5      n[         R                  " X45      nSU l        [         R                  " X%5      n[         R                  " Xf5      n[         R                  " XSU5      n[        R                  " XXU45      $ )Nrt  FT)r;   Ú	reductionrC  Ú
index_exprr*  r(  ÚtruedivÚsubÚmulr<   rz  )	rp   rz  r  Úsum_r)  ÚmeanÚdxÚdx2Úm2s	            r^   Úwelford_reduce_fallbackÚ"SIMDKernel.welford_reduce_fallbackñ  s�   € Ü�}Š}˜U¨5°%Ó8ˆØ %ˆÔÜ—’ §¡× =Ñ =¸uÓEˆÜ�{Š{˜4Ó(ˆà $ˆÔÜ�WŠW�UÓ!ˆÜ�gŠg�b‹oˆÜ�]Š]˜5¨°Ó4ˆÜ×!Ò! 4¨VÐ"4Ó5Ð5r`   c                óè   • [         R                  " XSU5      n[         R                  " X#5      n[         R                  " U5      n[         R                  " XSU5      n[        R
                  " X645      $ )NÚmaxrt  )r;   rÁ  rÄ  Úexpr<   rz  )rp   rz  r  ÚvmaxrÄ  rÏ  Úvsums          r^   Ú prepare_softmax_twopass_fallbackÚ+SIMDKernel.prepare_softmax_twopass_fallbacký  sT   € Ü�}Š}˜U¨5°%Ó8ˆÜ�gŠg�eÓ"ˆÜ�gŠg�c‹lˆÜ�}Š}˜U¨5°#Ó6ˆÜ×!Ò! 4 ,Ó/Ð/r`   c                ó   • [         erX   rx  rt   s    r^   Úcodegen_kernelÚSIMDKernel.codegen_kernel  r}  r`   c                ó   • g rX   rË   rt   s    r^   rë  ÚSIMDKernel.codegen_body  ó   € Ør`   c                ó   • g rX   rË   )rp   rÌ  s     r^   rö   Ú)SIMDKernel.codegen_iteration_ranges_entry
  rÙ  r`   )rw  rx  r=  r    rE  r*  r>  rC  rM  r³   rI  r<  rK  r@  rH  r´   r5  rT  rU  r9  rL  r(  rF  )NNNNF)r(  údict[str, sympy.Expr]r*  rJ   r›   rÝ   rV  úOptional[bool]rW  rÝ  rF  úOptional[dict[str, sympy.Expr]]rI  rˆ   r…   r†   )rb  rˆ   r…   z
str | None)rg  rÜ   r…   r   r  )rz  útorch.dtyper…   r   )r…   rß  rß   r‡   )r›   rÝ   rC  rˆ   ru   rˆ   r@  rÜ  rK  rˆ   r…   úlist[IterationRangesRoot])r›   zdict[str, str]r…   r†   )r¥  úSequence[sympy.Expr]r…   r†   )ri   r   r™   r‚   r  r@   r…   r†   )r…   r�   )r…   z	list[str])r™   r‚   r…   r‚   )r™   r‚   r8  r„   r…   r‚   )r…   z'contextlib.AbstractContextManager[None])r»   r‚   r…   r€   )r  úIterable[sympy.Expr]r»   úSequence[Sequence[sympy.Expr]]r…   zStuple[list[list[sympy.Expr]], list[list[Callable[[list[sympy.Expr]], sympy.Expr]]]])r  râ  r»   rã  r(  r‚   r…   rã  )r  râ  r»   rã  r(  r‚   r…   rˆ   )r»   rã  r…   úlist[list[sympy.Expr]])r  rá  r»   rã  r…   rä  )r™   r‚   r…   rˆ   )r™   r‚   r…   r   )r…   rà  )r¶   r‚   r…   r‚   rà   )NT)ri   r   r¨   zOptional[IRNode]rs  rˆ   r…   r†   )r�  zUnion[str, OpsWrapper]r  úUnion[int, float]r…   zIterator[str])r™   r‚   r…   r�   )r…   rÞ   )rÌ  r±   )GrŠ   r‹   rŒ   r�   rŽ   Úpexprr-  r"  r/  rh   rc  rh  rn  r’   r1   ru  r{  r€  rƒ  rJ  rŸ  rS  r¦  r«  rD  rG  rk   r»  rÂ  rÈ  rÐ  rË  r7  r6  rÜ  rñ  rõ  Ústaticmethodr$  Úclassmethodr�   r�   r‘   r)  r-  r2  r0  r>  rG  rK  r\  r`  rX  rh  rp  rt  rï  rð  r|  r…  r‰  r�  r¡  r¾  rË  rÒ  rÕ  rë  rö   r“   r”   r•   s   @r^   rƒ   rƒ   „  sÃ  ø‡ ñð */€EÐ&Ó.Ø&Ó&Ø!€O�TÓ!àÓð /3Ø8<Ø9=Ø9=Ø$)ðADà%ðADð %ðADð ,ð	ADð
 (6ðADð )7ðADð 7ðADð "ðADð 
÷ADð ADðFð  ðð 
ôô$%òð
 ØóJó ó ðJô"ô2ð óHó ðHôð5à+ð5ð ð5ð ð	5ð
 &ð5ð ð5ð 
#ô5ôn-ôô*ôôô
ôRô'ô
ô	ô'ô
ð$>Øð>Ø':ð>à	ô>ðØðØ':ðà	ôô(ô0
ð ðP1Ø$ðP1Ø/MðP1ð
óP1ó ðP1ðd ð
 ',§g¡g§k¡kð	à$ðð 0ðð $ð	ð
 
(ôó ðð& ð
 ',§g¡g§k¡kð	à$ðð 0ðð $ð	ð
 
ôó ðð VØ5ð Và	ô VðD ðOà$ðOð 0ðOð
 
 óOó ðOô84ô
ô,7ð$1àð$1ð 
ô$1ôL
ô
ô"<ò8ð OSð6Øð6Ø/ð6ØGKð6à	õ6ð
 ×Ñð)Ø*ð)Ø3Dð)à	ó)ó ð)ô&ð0 ñó ðô
(ò@òDFòP
6ò0ò"ò÷ò r`   rƒ   c                  óf  • \ rS rSr% Sr\rS\S'   S rS r	\	r
\	rS rS rS	 rS
 r S0   S1S jjrS r S2   S3S jjr  S4S jr\      S5S j5       r S6       S7S jjrS8S jr    S9S jrS rSS.S jr    S:S jrS;S jr      S<S jrSSS.   S=S jjrS r S6           S>S jjr S r!\"\#RH                  " S 5      S?S! j5       5       r%\"      S@S" j5       r&\"      SAS# j5       r'\"        SBS$ j5       r(\"  SCS% j5       r)\"          SDS& j5       r*\"        SES' j5       r+\"        SFS( j5       r,\"\-R\                  R^                  S4   SGS) jj5       r0\"\-R\                  R^                  S4   SHS* jj5       r1S+ r2SIS, jr3 SJ SKS- jjr4S. r5S/r6g)LÚSIMDSchedulingi  zc
Single Instruction Multiple Data parent class used for fusion across
multiple different backends.
z	type[Any]Úkernel_typec                ó&   • [        S U 5       5      $ )Nc              3  ó†   #   • U  H7  n[         R                  R                  R                  [	        U5      5      v •  M9     g 7frX   r6  rû   s     r^   rý   Ú*SIMDScheduling.group_fn.<locals>.<genexpr>  s-   é € ÐPÊ%ÀQ”Q—W‘W×%Ñ%×.Ñ.¬}¸QÓ/?×@Ð@Ê%ùs   ‚?A©rˆ  rÓ  s     r^   Úgroup_fnÚSIMDScheduling.group_fn  s   € ÜÑPÉ%ÓPÓPÐPr`   c                óÄ
  ^^^^• [        U[        R                  5      (       d  [        U[        R                  5      (       a  [        R                  R                  X5      $ UR                  u  nu  nmUR                  u  nu  mm[        X5      nUR                  5       (       a3  UR                  5       (       d  UR                  5       (       a  U" S5        OGUR                  5       (       a2  UR                  5       (       d  UR                  5       (       a  U" S5        UR                  5       (       aô  UR                  5       (       aß  UT:H  =(       a    TT:H  nU(       d  SSKJ	n  UR                  X5      nU(       d  U" SUTTT5        U(       a”  UR                  5       (       d  UR                  5       (       aj  UR                  5       (       d  X!p!U R                  UR                  5       UT5      m[        UU4S jUR                  5        5       5      (       d	  U" S5        gU$ UR                  5       (       GdÙ  UR                  5       (       GdÃ  UT:X  a  TT:X  d¢  UR                  5       (       d  U" SUTTT5        gUR                  5        Hl  nUR                  5       (       a    OVUR                  5       UR!                  5       -  (       d  MB  UR                  u  nu  pšXI:X  a  TU
:X  a  M`  U" S	UU	TU
5          g   X4 H  nUR                  5       (       d  M    g
   U R                  UR                  5       UT5      nU R                  UR                  5       UT5      nU R                  UR                  5       UR                  5       -   UT5      n["        R$                  R&                  (       a`  S
n[)        U5      S:”  a)  [)        U5      S:”  a  XÍs=:H  =(       a    U:H  Os  nOXÎ:H  nO[)        U5      S:”  a  XÞ:H  nU(       d  U" SUUU5        gg
UR                  5       (       dø  UR                  5       (       aã  TS:X  a  TS:w  d   eUTT-  :X  a¹  [        UU4S jUR                  5        5       5      (       d	  U" S5        g["        R$                  R*                  (       ag  UR                  5       (       dR  [-        U R                  UR                  5       U5      R/                  5       5      US4TTS44;   nU(       d  U" S5        U$ g
UT:w  a  U" S5        UT:H  $ UR                  5       (       a  UR                  5       (       a   eU R1                  X!5      $ )zŠ
Hook called by Scheduler to determine if the Triton backend
can fuse node1 and node2.  These nodes might already be
FusedSchedulerNodes.
z&Split scan cannot fuse with reductionsr   )ÚMixOrderReductionz1numel/rnumel mismatch (reduce) (%s, %s), (%s, %s)c              3  ó†   >#   • U  H6  n[         R                  TR                  5       UR                  5       TS 9v •  M8     g7f©)r(  N)rƒ   r-  r¦   Ú
get_ranges)rü   Ún2Úrnumel1r(  s     €€r^   rý   Ú*SIMDScheduling.can_fuse.<locals>.<genexpr>Q  s?   øé € ð ò 0˜ô ×,Ñ,ØŸ™›¨¯©«È'ð -õ ò 0ùs   ƒ>Az/invalid loop order and tiling for native matmulFz5numel/rnumel mismatch (non-reduce) (%s, %s), (%s, %s)z:numel/rnumel mismatch prologue mismatch (%s, %s), (%s, %s)Tr   ztiling mismatch (%s, %s, %s)r>   c              3  óp   >#   • U  H+  n[         R                  TT4UR                  5       5      v •  M-     g 7frX   )rƒ   r-  rö  )rü   rÙ   Únumel2Úrnumel2s     €€r^   rý   rù  ž  s3   øé € ð â.˜ô ×,Ñ,¨f°gÐ->ÀÇÁÃ×OÐOÚ.ùs   ƒ36z"nodes numel/rnumel incompatibilityzinvalid tiling for reductionznodes numel incompatibility)rÿ   r   ÚForeachKernelSchedulerNodeÚcan_fuser  r0   Úis_split_scanru   Útorch._inductor.schedulerró  rM  Úselect_tilingÚ	get_nodesr  Úis_templateÚused_buffer_namesÚget_buffer_namesr   r[   Ú tiling_prevents_pointwise_fusionr  Ú tiling_prevents_reduction_fusionrˆ  r¦   Úcan_fuse_horizontal)rp   Únode1Únode2r  Únumel1ÚwhyÚreduction_can_fuseró  r¨   Ú	pro_numelÚ
pro_rnumelrÙ   Útiling1Útiling2Útiling3ÚcondÚis_reduction_tiling_validrû  rø  rü  r(  s                    @@@@r^   rþ  ÚSIMDScheduling.can_fuse  sÕ  û€ ô �eœY×AÑA×BÑBÄjØ”9×7Ñ7÷G
ñ G
ô ×7Ñ7×@Ñ@ÀÓNÐNà$Ÿ{™{ÑˆÑˆF�GØ$Ÿ{™{ÑˆÑˆF�GÜ˜Ó%ˆà×Ñ× Ñ ¨×)<Ñ)<×)>Ñ)>Ø×!Ñ!×#Ñ#ÙÐ<Ô=øØ× Ñ ×"Ñ"¨5×+>Ñ+>×+@Ñ+@Ø×!Ñ!×#Ñ#ÙÐ<Ô=à×Ñ×Ñ E×$6Ñ$6×$8Ñ$8Ø!'¨6Ñ!1×!H°gÀÑ6HÐÞ%ÝGà%6×%?Ñ%?ÀÓ%MÐ"æ%ÙØGØØØØôö "Ø×&Ñ&×(Ñ(¨E×,BÑ,B×,DÑ,Dð ×-Ñ-×/Ñ/Ø#(˜5ð ×+Ñ+¨E¯O©OÓ,=¸vÀwÓO�Üõ ð $Ÿo™oÔ/ó	÷ ñ ñ ÐIÔJØ à%Ð%à×!Ñ!×#Ò#¨E×,>Ñ,>×,@Ò,@Ø˜fÓ$¨°GÓ);Ø×(Ñ(×*Ñ*ÙØOØØØØôð !ð !&§¡Ö 1˜à×+Ñ+×-Ñ-Ù!ð  $×5Ñ5Ó7¸%×:PÑ:PÓ:R×RÙ$Ø59·Z±ZÑ2˜Ñ2˜IØ &Ó 3¸À:Õ8MÙØ \Ø &Ø )Ø 'Ø *ôñ $)ñ# !2ð& “^�Ø—=‘=—?“?Ùñ $ð
 ×(Ñ(¨¯©Ó):¸FÀGÓLˆGØ×(Ñ(¨¯©Ó):¸FÀGÓLˆGØ×(Ñ(Ø—‘Ó! E§O¡OÓ$5Ñ5°v¸wóˆGô �}‰}×=×=Ø�Ü�w“< !Ó#Ü˜7“| aÓ'Ø&×<Ó<°WÔ<™à&Ñ1™Ü˜“\ AÓ%Ø"Ñ-�DÞÙØ6ØØØô	ð !àà×!Ñ!×#Ñ#¨×(:Ñ(:×(<Ñ(<Ø˜a“< G¨q£LÐ0Ð0Ø˜ 'Ñ)Ó)Üõ à"Ÿ_™_Ô.ó÷ ñ ñ Ð<Ô=Ø ä—M‘M×B×BØ!×-Ñ-×/Ñ/ä05Ø×*Ñ*¨5¯?©?Ó+<¸fÓE×LÑLÓNó1ð   ˜Ø ¨!Ð,ðñ1Ð-ö 5ÙÐ:Ô;Ø4Ð4Øà˜ÓÙÐ1Ô2Ø˜VÑ#Ð#à×!Ñ!×#Ñ#¨E×,>Ñ,>×,@Ñ,@Ð@Ð@à×'Ñ'¨Ó5Ð5r`   c           
     óæ  ^^^^^^^• / m[         [        R                     " 5       m[        5       m[        5       mS mUU4S jnUU4S jnU4S jnUUUU4S jn[        R                  UUUU4S j5       nUU4S jn	U HÌ  n
U
T;   a  M  TR                  U
5        U" U
5      (       aT  U	" U
T5      (       a  U" 5           S S S 5        T(       a"  U" U
5      (       d  T=(       d    [        T5      mOS mU" U
5        M}  U" U
5      (       a#  U" 5          TR                  U
5        S S S 5        M­  [        ST ST S	U
R                  S
    35      e   T$ ! , (       d  f       N˜= f! , (       d  f       Mò  = f)Nc                ó~   >• U R                   u  nu  p#UT:H  =(       a    UT:H  =(       d    UTT-  :H  =(       a    US:H  $ ©Nr>   ©r  ©rÙ   r  Ú
node_numelÚnode_rnumelrl   r)  s       €€r^   Úfits_in_main_bodyÚ@SIMDScheduling.generate_node_schedule.<locals>.fits_in_main_bodyÇ  sF   ø€ Ø+,¯7©7Ñ(ˆAÑ(�
Ø %Ñ'×A¨K¸6Ñ,A÷ Ø˜e f™nÑ,×A°ÀÑ1Aðr`   c                ó`   >• U R                   u  nu  p#UT:H  =(       a    US:H  =(       a    TS:g  $ r  r  r  s       €€r^   Úfits_outside_reductionÚESIMDScheduling.generate_node_schedule.<locals>.fits_outside_reductionÍ  s2   ø€ Ø+,¯7©7Ñ(ˆAÑ(�
Ø Ñ&×K¨;¸!Ñ+;×KÀÈ!ÁÐKr`   c                ód   >• U R                   R                   H  nUR                  T;   d  M    g   gr,  )Úread_writesÚreadsri   )rÙ   ÚreadÚcurrent_loop_buffer_usages     €r^   Úexpect_improved_memory_usageÚKSIMDScheduling.generate_node_schedule.<locals>.expect_improved_memory_usageÑ  s,   ø€ ØŸ™×+Ô+�Ø—9‘9Ð 9Õ9Ùñ ,ð r`   c                óº  >• TR                  U 5        TR                  U 5        TR                  U R                  R                   Vs/ s H  oR
                  PM     sn5        U R                  5       (       a›  [        U [        R                  5      (       a|  [        U R                  [        R                  5      (       aS  [        U R                  R                  [        R                  5      (       d   TR                  U R                  5       5        g TR                  U R                  R                    Vs/ s H  oR
                  PM     sn5        g s  snf s  snf rX   )rÓ   rµ   Úupdater#  r$  ri   ru   rÿ   r   rO  r¨   r   rP  ÚdataÚScanÚget_nameÚwrites)rÙ   rT   r&  Údoner&  Únot_ready_yet_nodess     €€€€r^   Úschedule_node_in_loopÚDSIMDScheduling.generate_node_schedule.<locals>.schedule_node_in_loop×  sä   ø€ Ø�H‰H�QŒKØ× Ñ  Ô#Ø%×,Ñ,¸a¿m¹m×>QÒ>QÓ-RÒ>Q¸¯f¬fÑ>QÑ-RÔSð
 —‘× Ñ Ü˜q¤)×"9Ñ"9×:Ñ:Ü˜qŸv™v¤r×'8Ñ'8×9Ñ9Ü" 1§6¡6§;¡;´·±×8Ñ8à#×'Ñ'¨¯
©
«Õ5à)×0Ñ0À!Ç-Á-×BVÒBVÓ1WÒBV¸Q·&´&ÑBVÑ1WÕXùò .Sùò 2Xs   ÁEÄ6Ec               3  ób  >#   • T(       a  TS   [         L a  TR                  5         OTR                  [        5        T(       a1  TR	                  T[        5        TR	                  TS-   [         5        S mS v •  TR                  [         5        TR                  5         T R                  5         g 7f)Nré  r>   )rG   Úpoprµ   rF   ÚinsertÚclear)r&  Úmaybe_split_indexr&  r0  s   €€€€r^   Úend_current_reduction_loopÚISIMDScheduling.generate_node_schedule.<locals>.end_current_reduction_loopè  sŒ   øé € ö  ¨rÑ!2´oÒ!EØ×!Ñ!Õ#à×$Ñ$Ô%5Ô6Þ Ø×$Ñ$Ð%6Ô8HÔIØ×$Ñ$Ð%6¸Ñ%:¼OÔLØ$(Ð!ÛØ× Ñ ¤Ô1Ø×%Ñ%Ô'Ø%×+Ñ+Õ-ùs   ƒB,B/c                ó    >• TS:X  a  gTU R                   -  (       d  gU(       a  [        US   [        [        45      (       a   e[	        T5      $ )Nr>   Fré  )Ú	ancestorsrÿ   rG   rF   rˆ   )r¨   r&  r0  r)  s     €€r^   Ú#requires_closing_previous_reductionÚRSIMDScheduling.generate_node_schedule.<locals>.requires_closing_previous_reductionø  sS   ø€ Ø˜‹{ØØ&¨¯©×7ØÞ ¬Ø˜bÑ!¤OÔ5EÐ#F÷*ñ *ð ð ô Ð+Ó,Ð,r`   zunexpected group: (r¢   z) != r>   )
r   r   r.   rï  rð  rÓ   r  rµ   ry  r  )rp   rš   rl   r)  r  r   r'  r1  r8  r<  r¨   r&  r/  r7  r&  r0  s     ``       @@@@@r^   Úgenerate_node_scheduleÚ%SIMDScheduling.generate_node_schedule¾  sT  þ€ Ø#%ˆÜœ)×5Ñ5Ò6Ó8ˆô 0:«|ÐÜ5?³\Ð!Ø+/Ðö	ö	Lõ	÷	Yð 	Yô" 
×	"Ñ	"÷	.ó 
#ð	.ö	-ó ˆDØ�t‹|ÙØ�H‰H�TŒNá  ×&Ñ&Ù6°t¸]×KÑKÙ3Õ5Ø÷ 6ö -Ñ5QÐRV×5WÑ5Wà(9×(O¼SÀÓ=OÑ%ð )-Ð%á% dÖ+Ù'¨×-Ñ-Ù/Õ1Ø!×(Ñ(¨Ô.÷ 2Ñ1ô *Ø)¨%¨°°6°(¸%ÀÇ
Á
È1Á¸ÐOóð ñ- ð4 Ð÷' 6Õ5ú÷ 2Ö1ús   Â<EÄE!Å
E	Å!
E0	c                óà   • UR                   UR                  p2UR                  UR                  5       -  (       d"  UR                  UR                  5       -  (       a   eU R	                  X#5        g rX   )r	  r
  r;  Úget_operation_namesÚ_codegen_mix_order_reduction)rp   r¨   r	  r
  s       r^   Úcodegen_mix_order_reductionÚ*SIMDScheduling.codegen_mix_order_reduction  sW   € Ø—z‘z 4§:¡:ˆuð —O‘O e×&?Ñ&?Ó&A×AØ�O‰O˜e×7Ñ7Ó9×9ð	
ð 
ð 	×)Ñ)¨%Õ7r`   c                ó´   • UR                  5       n/ n/ nU H<  nUR                  5       (       a  UR                  U5        M+  UR                  U5        M>     X44$ rX   )r  ru   rµ   )rp   r¨   rš   Ú
reductionsÚ	epiloguess        r^   Ú#_split_mix_order_reduction_epilogueÚ2SIMDScheduling._split_mix_order_reduction_epilogue(  sX   € à—‘Ó ˆØˆ
Øˆ	ÛˆDØ× Ñ ×"Ñ"Ø×!Ñ! $Ö'à× Ñ  Ö&ñ	 ð
 Ð$Ð$r`   c           	     ó  • UR                   UR                  pTUR                  nU R                  UXES./USSSS.5      S   nUR                  (       d   eUR
                  (       d   eX'l        U R                  Xg5        UR                  R                  [        UR                  5      UR                  S   -  UR                  S   UR                  -   S-
  UR                  -  -  S	[        R                  S
9u  p‰n
U
S:X  d   SU
< 35       eU   UR                  5         SSS5        [         R"                  " 5       n[$        R&                  " U5         U   U(       a#  UR)                  [*        R,                  " SS95        UR/                  5       nSSS5        SSS5        U(       a)  WR1                  [3        [4        R6                  5      S5      nXyW4$ ! , (       d  f       N»= f! , (       d  f       N[= f! , (       d  f       Nd= f)z
for_benchmark:
    True if the generated code is for benchmarking. We need make
    sure benchmark harness code is generated.
)rT   rU   NT)r*  rF  rI  rV  r   rU   rT   r>   F)rz  zws_off=)Úbenchmark_kernelÚtriton_)rl   r(  r&  Úcreate_kernel_choicesrH  rI  rT  Ú!codegen_node_schedule_with_kernelr  Ú	workspacer  rU  r@  rY   r  rë  rï  Ú	ExitStackr=   Úset_kernel_handlerÚenter_contextr   ÚpatchrÕ  Úreplacer   r5   ÚKERNEL_NAME)rp   Úkernel_featuresÚ
split_sizeÚfor_benchmarkrl   r)  r&  rn   r  Úws_nameÚws_offÚstackÚsrc_codes                r^   Ú-_generate_kernel_code_for_mix_order_reductionÚ<SIMDScheduling._generate_kernel_code_for_mix_order_reduction4  sÈ  € ð (×-Ñ-¨×/NÑ/NˆvØ'×5Ñ5ˆà×+Ñ+ØØÑ(Ð)à+Ø!%Ø'+Ø15ñ	ó	
ð ñ	ˆð ×*×*Ð*Ð*Ø×)×)Ð)Ð)Ø'ÔØ×.Ñ.¨}ÔEð $Ÿ[™[×2Ñ2Ü�×/Ñ/Ó0Ø�m‰m˜EÑ"ñ#à—‘˜cÑ" V×%7Ñ%7Ñ7¸!Ñ;À×@RÑ@RÑRñTð Ü—+‘+ð 3ð 
Ñˆ�Fð ˜‹{Ð(˜w˜v™i˜LÓ(ˆ{ÚØ×ÑÔ!÷ ô ×$Ò$Ó&ˆÜ×!Ò! &Õ)ª5ÞØ×#Ñ#¤F§L¢LÀ$Ñ$GÔHØ×,Ñ,Ó.ˆH÷ ,1×)ö
 ð
  ×'Ñ'¬¬K×,CÑ,CÓ(DÀiÓPˆHØ Ð(Ð(÷ �Vú÷ ,1­5ú×)Õ)ús0   ÄGÅG6Å;G%ÆG6Ç
G"Ç%
G3	Ç/G6Ç6
HNc                ó   • [         erX   rx  )rp   ÚmodÚn_spills_thresholdÚ
node_namess       r^   Úbenchmark_codegened_moduleÚ)SIMDScheduling.benchmark_codegened_moduleh  s
   € ô "Ð!r`   c                ót  ^ ^^!^"• [         R                  R                  T5      u  m"nUU"4S jnU" 5       n[        =R                  S-  sl        T R                  U5      u  pg/ nU HD  n	U	R                  5         U	R                  5       n
U
R                  5         UR                  U
5        MF     T R                  TR                  5       U-   T"U5      n[        UT"U5      m![        R                  R                  R                   (       dƒ  [        R"                  R$                  ch  [        R"                  R&                  (       d*  [        R(                  (       d  [        R*                  (       a  U!U 4S jn[,        R.                  " UUS5      nT R1                  T!USS9u  pÞn[3        US   R4                  R6                  5      n0 nU(       Gac  U GH  n	U	R9                  5       S   R4                  R;                  5       nU	R9                  5       S   R<                  S   R4                  R9                  5       S   R4                  R;                  5       nUUU'   T R                   (       d   eT R                   R>                  RA                  U	R9                  5       S   R<                  S   R4                  R;                  5       5        [B        RD                  RF                  RA                  U5        GM!     URH                   H.  nURK                  URL                  URL                  5      Ul&        M0     T RO                  XûU5      nUUl(        [S        U5      Ul)        [B        RT                  " U5         T!RW                  5        HD  nUR9                  5       S   R4                  R;                  5       U;  d  M4  URY                  5         MF     S S S 5        [B        RD                  RZ                  R]                  S5        T R_                  US 5        URa                  URP                  SS	9  [B        RD                  =RF                  URF                  -  sl#        [B        RD                  =Rb                  URb                  -  sl1        [e        U5      [e        URH                  5      :X  d   e[B        RD                  RZ                  Rg                  T"U-   S-
  U-  5      n[i        URH                  5       GH  u  nnURL                  nU S
U 3nU S
U 3nSU SU 3nSSS.nURK                  URj                  URj                  5      nU SU SU SU SU SU SU S3n[B        RD                  Rm                  U5      =n [        Rn                  :w  a	  USU  S3-  n[B        RD                  RZ                  Rq                  U5        [B        RD                  RZ                  Rr                  RA                  U5        GM     URu                  5         U(       a  T Rw                  U5        T Ry                  5         g ! , (       d  f       GNj= f)Nc                 ót  >• [         R                  R                  b  [         R                  R                  $ [        R                  " TR                  5       5      n U R                  nUS-  n[        R                  R                  R                  T5      n[        [        X2-  5      S5      n[        US5      nU$ )Né   é   é€   )r   r[   Úmix_order_reduction_split_sizer*   ÚcreateÚ
get_deviceÚmulti_processor_countr=   r®   r¯   r  rÎ  r,   Úmin)Údevice_propÚnum_smÚestimated_num_splitsÚ
numel_hintrW  r	  rl   s        €€r^   Ú_pick_split_sizeÚESIMDScheduling._codegen_mix_order_reduction.<locals>._pick_split_sizep  s–   ø€ ä�}‰}×;Ñ;ÑGÜ—}‘}×CÑCÐCô +×1Ò1°%×2BÑ2BÓ2DÓEˆKØ ×6Ñ6ˆFØ#)¨A¡:Ð ô Ÿ™×)Ñ)×3Ñ3°EÓ:ˆJÜœ_¨ZÑ-OÓPÐRTÓUˆJÜ˜Z¨Ó-ˆJØÐr`   r>   c                ó‚   >• TR                  TU SS9u    p[        R                  " U5      nTR                  U5      u  pAU$ )NT©rW  rX  )r]  r!   Úloadrc  )Úcandidate_split_sizer  r\  r`  ÚmsrV  rp   s        €€r^   Ú_benchÚ;SIMDScheduling._codegen_mix_order_reduction.<locals>._bench   sO   ø€ Ø!%×!SÑ!SØ#Ø3Ø"&ð "Tð "‘��1ô
 "×&Ò& xÓ0�Ø×7Ñ7¸Ó<‘�Ø�	r`   rg  Frv  r   z!# Call mix order reduction kernel)rs  z * Ú(z + 1) * ÚaminÚamax)rn  rÎ  z = r¿  z : z].view(r¢   z).z(dim=0)z.to(rë   )=r   ró  Úget_numel_rnumelr   rC  rH  Úcancel_reduction_splitÚextract_pw_from_reductionÚswap_pw_red_dimensionrµ   r>  r  rJ   rY   rZ   r   Údeterministicr[   rj  Ú'mix_order_reduction_autotune_split_sizeÚmax_autotuneÚcoordinate_descent_tuningr)   Úautotune_single_fieldr]  rˆ   r¨   Ú_split_sizeÚget_outputsr-  ÚusersÚremoved_opsrÓ   r=   r®   Úremoved_buffersrU  rÖ   r   Údefine_kernelr0  r    rQ  Úscheduler_nodesÚmark_runrk  Úmake_commentÚcodegen_commentrt  Úinplaced_to_remover  Úcodegen_python_sizevarr‘  r!  r—  r  Ú	writelineÚ	allocatedrp  Ú_codegen_nodesÚfree_buffers_in_scheduler)#rp   r	  r
  r)  rs  rW  Únode2_reductionsÚnode2_epilogueÚconverted_nodesÚsubnodeÚ	convertedr&  rz  rn   rY  r\  Úis_split_reductionÚrenameÚbufnameÚusernameÚpartial_accumr0  r¨   Únsplitr“  r   Ú
stride_strrk  ÚendÚreduction_type2opÚopnameÚfinal_reduceÚbuffer_dtyperV  rl   s#   ``                               @@r^   rB  Ú+SIMDScheduling._codegen_mix_order_reductionm  sD  û€ Ü!×3Ñ3×DÑDÀUÓK‰ˆˆvö	ñ  &Ó'ˆ
ô 	×+Ò+¨qÑ0Õ+ð ,0×+SÑ+SØó,
Ñ(Ðð ˆÛ'ˆGØ×*Ñ*Ô,Ø×9Ñ9Ó;ˆIØ×+Ñ+Ô-Ø×"Ñ" 9Ö-ñ	 (ð
 ×3Ñ3Ø�O‰OÓ Ñ/°¸ó
ˆô -¨]¸EÀ6ÓJˆô —‘×&Ñ&×4×4Ü—‘×<Ñ<ÑDä—‘×E×EÜ×&×&Ü×3×3öô '×<Ò<ØØØóˆJð %)×$VÑ$VØØ!Øð %Wð %
Ñ!ˆ˜ô "Ð"2°1Ñ"5×":Ñ":×"FÑ"FÓGÐØˆßÜ+�Ø!×-Ñ-Ó/°Ñ2×7Ñ7×@Ñ@ÓB�à×'Ñ'Ó)¨!Ñ,ß‘U˜1ñç‘TŸ+™+›-¨ñ+÷ ‘TŸ(™(›*ð	 ð #+��w‘Ø—~—~Ð%�~Ø—‘×*Ñ*×.Ñ.Ø×'Ñ'Ó)¨!Ñ,×2Ñ2°1Ñ5×:Ñ:×CÑCÓEôô —‘×'Ñ'×+Ñ+¨G×4ñ ,ð "(×!@Ô!@�Ø,2¯J©JØ!×-Ñ-¨}×/HÑ/Hó-�Ö)ñ "Að
 ×(Ñ(¨À&ÓIˆØ(ˆÔÜ$ XÓ.ˆÔä×!Ò! &Õ)Ø'×7Ñ7Ö9�ð ×#Ñ#Ó% aÑ(×-Ñ-×6Ñ6Ó8ÀÕFØ—M‘M–Oñ :÷ *ô 	
�‰×Ñ×)Ñ)Ð*MÔNØ×Ñ˜]¨DÔ1à×Ñ˜6×-Ñ-¸UÐÑCÜ	�‰×Ò 6×#9Ñ#9Ñ9ÕÜ	�‰×"Ò" f×&?Ñ&?Ñ?Õ"ô �?Ó#¤s¨6×+JÑ+JÓ'KÓKÐKÐKÜ—‘×%Ñ%×<Ñ<Ø�ZÑ !Ñ#¨
Ñ2ó
ˆô #,¨F×,KÑ,K×"LÑˆC�Ø'×3Ñ3ˆKà"˜8 3 v hÐ/ˆJØ�e˜3˜z˜lÐ+ˆEØ�c�U˜( : ,Ð/ˆCàØñ!Ðð '×*Ñ*Ø×,Ñ,¨m×.JÑ.JóˆFð *˜]¨#¨g¨Y°a¸°w¸cÀ#ÀÀgÈfÈXÐUWÐX^ÐW_Ð_aÐbhÐaiÐipÐqˆLô !"§¡× 1Ñ 1°+Ó >Ð>�Ä5Ç;Á;ÓNØ $ | n°AÐ 6Ñ6�Ü�G‰G× Ñ ×*Ñ*¨<Ô8ô �G‰G× Ñ ×*Ñ*×.Ñ.¨{×;ñ- #Mð0 	×$Ñ$Ô&æØ×Ñ Ô/à×&Ñ&Õ(÷c *Ö)ús   Í,AX(Î1X(Ø(
X7c                ór  • U R                   (       d   eU Vs/ s H.  o3R                  5       U R                   R                  ;  d  M,  UPM0     nnU(       d  g [        US S9R                  u  nu  pVU R                  XU5      n[        R                  SU5        U R                  [        XuXb5      5      $ s  snf )Nc                ó4   • [        U R                  5       5      $ rX   ©rÜ   ru   ©rT   s    r^   rÌ   Ú/SIMDScheduling._codegen_nodes.<locals>.<lambda>  s   € ´c¸!¿.¹.Ó:JÔ6Kr`   rÎ   zSchedule:
 %s)
r   r-  r‹  rÎ  r  r>  Úschedule_logÚdebugÚcodegen_node_schedulerJ   )rp   rš   Úcoalesce_analysisr¨   r  rl   r)  r&  s           r^   r–  ÚSIMDScheduling._codegen_nodes  s¨   € ð
 �~�~Ðˆ~á"ó
Ú"�T§m¡m£o¸T¿^¹^×=WÑ=WÑ&W�D™Uð 	ð 
ö ØÜ  Ñ,KÑL×RÑRÑˆ‰?ˆEà×3Ñ3°EÀ&ÓIˆÜ×ÑÐ+¨]Ô;à×)Ñ)Ü˜}°VÓOó
ð 	
ùò
s   ˜+B4ÁB4c                ó>  • U R                   (       d   eUR                  5        Vs/ s H/  nUR                  5       U R                   R                  ;  d  M-  UPM1     nn[	        U5      S:X  a  g[
        R                  R                  R                  R                  (       af  [	        U5      [	        WR                  5       5      :w  a4  U R                   (       d   e[         R                  " U R                   U5      n[        U5      nOSnU R                  X#5      $ s  snf )z;
Given a set of pre-fused nodes, generate a Triton kernel.
r   N)r   r  r-  r‹  r  rY   rZ   r   r[   Úcoalesce_tiling_analysisÚFusedSchedulerNoder   r–  )rp   r¨   rš   r²  s       r^   Úcodegen_nodeÚSIMDScheduling.codegen_node  sÛ   € ð �~�~Ðˆ~ð Ÿ™Ô(ó
â(�Ø�}‰}‹ d§n¡n×&@Ñ&@Ñ@÷ Ù(ð 	ð 
ô
 ˆu‹:˜‹?Øä�?‰?×!Ñ!×(Ñ(×A×AÜ�5‹zœS §¡Ó!1Ó2Ó2Ø—~—~Ð%�~Ü ×3Ò3°D·N±NÀEÓJ�Ü 9¸$Ó ?Ñà $Ðà×"Ñ" 5Ó<Ð<ùò!
s   ¦,DÁDc                ó:  • [         R                  " [         R                  5      R                  n[	        U 5      (       d  gU Vs/ s H8  nUR                  5       (       d  M  UR                  5       R                  5       PM:     nnU H”  nUR                  5       (       a  M  [        U[        R                  5      (       d  M;  UR                  5       nUU Vs/ s H8  nUR                  5       (       d  M  UR                  5       R                  5       PM:     sn-  nM–     [        S U 5       5      (       d  g[        R                  R                  R!                  X5        U H,  n[        R                  R                  R!                  Xb5        M.     gs  snf s  snf )NFc              3  ó8   #   • U  H  n[        U5      v •  M     g 7frX   )r2   )rü   r  s     r^   rý   Ú8SIMDScheduling.can_use_32bit_indexing.<locals>.<genexpr>P  s   é € ÐFºI°DÔ)¨$×/Ð/ºIùrs  T)rY   ÚiinfoÚint32rÎ  r2   Úhas_tensor_outputrª  Ústorage_sizerÿ   r   ÚMutationOutputÚget_mutation_buffersr  r=   r®   r¯   Ú	check_leq)rl   ÚbuffersÚint_maxr¶  Ú	buf_sizesÚmutated_bufsr  s          r^   Úcan_use_32bit_indexingÚ%SIMDScheduling.can_use_32bit_indexing3  sG  € ô —+’+œeŸk™kÓ*×.Ñ.ˆä% e×,Ñ,Øñ ó
â�Ø×$Ñ$×&ó ,ˆC�N‰NÓ×)Ñ)Ö+Ùð 	ð 
ó ˆCØ×(Ñ(×*Ó*¬z¸#¼r×?PÑ?P×/QÓ/QØ"×7Ñ7Ó9�Øá+óâ+˜Ø×,Ñ,×.ó 4�C—N‘NÓ$×1Ñ1Ö3Ù+ññ ’	ñ ô ÑF¹IÓF×FÑFØô 	
�‰×Ñ×"Ñ" 5Ô2ÛˆDÜ�G‰G×Ñ×&Ñ& tÖ5ñ àùò/
ùòs   ÁFÁ!"FÃFÃ4"FFc                ó¢  • U R                  X!5        U(       dL  [        R                  " U5         [        R                  " U5       H  nUR                  5         M     SSS5        [        R                  =R                  UR                  -  sl        [        R                  =R                  UR                  -  sl        g! , (       d  f       Ni= f)zU
Process a kernel by generating code for its node schedule and updating graph state.
N)	rN  r=   rQ  rI   rŒ  r�  r®   rŒ  r’  )rp   rn   r&  rb  r¨   s        r^   Úprocess_kernelÚSIMDScheduling.process_kernelZ  sˆ   € ð 	×.Ñ.¨}ÔEÞ Ü×%Ò% fÕ-Ü.×9Ò9¸-ÖH�DØ—M‘M–Oñ I÷ .ô 	
�‰×Ò 6×#9Ñ#9Ñ9ÕÜ	�‰×"Ò" f×&?Ñ&?Ñ?Ö"÷	 .Õ-ús   ¯.C Ã 
Cc                óì  • UR                   nU R                  UUR                  UR                  UR                  5      u  p4U R                  UU/XS.5      nU H  nU R                  X&5        M     [        R                  " U5        U Hp  n[        R                  " U5         UR                  5       nSSS5        U R                  WX&5      n[        R                  SU5        X†l        [!        U5      Ul        Mr     A[#        U5      S:”  a  [        U5      n	OUu  n	[        R                  " U	5         UR%                  5        H  n
U
R'                  5         M     SSS5        U V
s/ s H  n
[)        U
[*        5      (       d  M  U
PM     nn
U R-                  X¹R                  5        [.        R0                  R2                  (       a\  [        R4                  R6                  R9                  5         [        R4                  R6                  R;                  U	R                  U5        U	R=                  U	R                  5        [.        R0                  R2                  (       a(  [        R4                  R6                  R?                  5         [.        R@                  (       a  U	RC                  5         [.        RD                  (       a  U	RE                  US   R                  5        [        R4                  =RF                  U	RF                  -  sl#        [        R4                  =RH                  U	RH                  -  sl$        [        R4                  R6                  RJ                  (       aÝ  [.        RL                  (       aÈ  US   RN                  RQ                  5       nUR%                  5        H—  n
U
RS                  5       nXÜ;  a  M  U
RT                  c   eU
RT                  RW                  5       nUc  MH  [X        S   S==   S-  ss'   [        R4                  R6                  R[                  SUR\                  < S	U S
35        M™     U R_                  5         g! , (       d  f       GN¡= f! , (       d  f       GN= fs  sn
f )z,
Generate code for nodes in kernel_features
)r*  rF  Nz+Generating kernel code with kernel_name: %sr>   r   ÚinductorÚintermediate_hookszrun_intermediate_hooks(r¢   rë   )0r&  Úget_tiling_and_scoresrl   r(  r²  rM  rN  rD   Úmerge_workspaces_inplacer=   rQ  rÕ  r�  r­  r°  r0  r    r  rŽ  r�  rÿ   r.   r‘  r   ÚcppÚenable_kernel_profiler®   rk  Ú write_kernel_context_guard_beginÚwrite_kernel_context_guardrt  Úwrite_kernel_context_guard_endÚnan_assertsrh  r¾  rŒ  r’  Úsupports_intermediate_hooksÚgenerate_intermediate_hooksr  Úlive_output_buffersr-  r¨   Úget_origin_noder   r”  ri   r—  )rp   rV  r&  r(  Útiling_scoreÚkernelsrn   r\  r0  Úfinal_kernelr¨   Úbase_scheduler_nodesÚ	live_outsri   Úorigin_nodes                  r^   r±  Ú$SIMDScheduling.codegen_node_schedulek  sV  € ð (×5Ñ5ˆà#×9Ñ9ØØ×!Ñ!Ø×+Ñ+Ø×-Ñ-ó	 
Ñˆð ×,Ñ,ØØˆHØ(ÑHó
ˆó
 ˆFØ×2Ñ2°=ÖIñ ä×,Ò,¨WÔ5ÛˆFÜ×%Ò% fÕ-Ø!×0Ñ0Ó2�÷ .à×,Ñ,¨X°}ÓMˆKÜ�I‰IÐCÀ[ÔQØ!,ÔÜ(¨Ó2ˆFÖñ ð ô ˆw‹<˜!ÓÜ& wÓ/‰Là%‰Oˆ\ä×!Ò! ,Õ/Ø'×7Ñ7Ö9�Ø—‘–ñ :÷ 0ñ +ó 
Ú*�T¬j¸Ô?P×.Q�D™]ð 	ð  
ð 	×ÑÐ1×3KÑ3KÔLÜ�:‰:×+×+Ü�G‰G× Ñ ×AÑAÔCÜ�G‰G× Ñ ×;Ñ;Ø×(Ñ(Ø$ôð 	× Ñ  ×!9Ñ!9Ô:Ü�:‰:×+×+Ü�G‰G× Ñ ×?Ñ?ÔAä××Ø×*Ñ*Ô,Ü×!×!Ø×(Ñ(¨°©×)?Ñ)?Ô@ä	�‰×Ò <×#?Ñ#?Ñ?ÕÜ	�‰×"Ò" l×&EÑ&EÑEÕ"ô �G‰G× Ñ ×<×<Ü×2×2ð   ™
Ÿ™×;Ñ;Ó=ˆIØ'×7Ñ7Ö9�Ø—}‘}“�ØÓ(ÙØ—y‘yÑ,Ð,Ð,Ø"Ÿi™i×7Ñ7Ó9�ØÓ*Ü˜ZÑ(Ð)=Ó>À!ÑCÓ>Ü—G‘G×(Ñ(×2Ñ2Ø1°+×2BÑ2BÑ1EÀRÈÀvÈQÐOöñ :ð 	×&Ñ&Õ(÷y .Ö-ú÷ 0Ö/üò
 
s$   Â"QÄ3(QÅ'Q1ÆQ1Ñ
Q	Ñ
Q.c                ó(   • U R                   " U0 UD6/$ rX   )rë  )rp   rV  Úkernel_argsÚkernel_kwargss       r^   rM  Ú$SIMDScheduling.create_kernel_choices¾  s'   € ð ×ÒØðàñð
ð 	
r`   c           	     ó^  • U   [         R                  " 5       n0 nU HÂ  nU[        L a!  UR                  UR	                  5       5        M-  U[
        L a  UR                  5         MH  UR                  5         UR                  UR                  5       5      nUR                  [        R                  UR                  R                  U5      R                  5       5      5        MÄ     UR!                  UR#                  5       5        U H�  nU[        L a!  UR                  UR	                  5       5        M-  U[
        L a  UR                  5         MH  [%        UR                  5        UR                  UR                  5       5      nUR'                  U5        M�     S S S 5        g ! , (       d  f       g = frX   )rï  rP  rF   rR  rñ  rG   ÚcloseÚdecide_inplace_updater2  rö  r*  r'  ÚfromkeysÚ_bodyÚindexing_from_argsr¦   r¦  Úkeysr(   rè   )rp   r&  rn   r[  Úall_indexingr¨   rÑ   s          r^   rN  Ú0SIMDScheduling.codegen_node_schedule_with_kernelÈ  s>  € ÚÜ×(Ò(Ó*ˆEØˆLó &�ØÔ+Ò+Ø×'Ñ'¨×(@Ñ(@Ó(BÖCØœ_Ò,Ø—K‘K–Mà×.Ñ.Ô0Ø!'×!<Ñ!<¸T¿_¹_Ó=NÓ!O�JØ ×'Ñ'ÜŸ™Ø ŸJ™J×9Ñ9¸*ÓE×LÑLÓNóöñ &ð ×$Ñ$ \×%6Ñ%6Ó%8Ô9ó &�ØÔ+Ò+Ø×'Ñ'¨×(@Ñ(@Ó(BÖCØœ_Ò,Ø—K‘K–Mô 6°d·j±jÔAØ!'×!<Ñ!<¸T¿_¹_Ó=NÓ!O�JØ—L‘L Ö,ñ &÷- �VŽVús   ƒFFÆ
F,©rb  c               ód
  • 0 nUR                  5       n/ n	U H…  n
U
R                  5       nU	R                  U
5        X¸-  (       d  M/  [        U5      S:X  d   eX—[	        [        U5      5      '   UR                  R                  [	        [        U5      5      5        / n	M‡     [        U	5      S:X  d   eUR                  U UUUU[        UU5      nUb  U$ U   U(       d  U/UQ H  nUR                  5         M     U" 5       nUR                  5       n[        U5       Hˆ  nUR                  U5      nUR                  U5         U H1  nUR                  UR!                  UR#                  5       5      5        M3     UR$                  R'                  [)        5       5        SSS5        MŠ     UR*                  R-                  5        GH@  u  nnSU S3nUR/                  UR1                  5       / 5      =n	(       d  M6  [3        S U	 5       5      n[4        R6                  " SU(       + 5         UR                  U5         U	 H�  n[        UR                  5       5      S:X  aB  [        U	5      S:X  a3  [        U5      (       a#  U=R8                  UR                  5       -  sl        UR                  UR!                  UR#                  5       5      5        M’     UR$                  R'                  [)        5       5        SSS5        SSS5        GMC     SSS5        [:        R<                  " U5         [?        W[@        5      (       d]  [B        RD                  RG                  URH                  RJ                  5         URM                  S5        SSS5        URM                  S	S
S9  UR*                   H  nSU S3nURM                  US
S9  M     UR                  5       n[        U5       H%  nUR                  U5      nURM                  U5        M'     [?        U[@        5      (       a  UnOURO                  5       n/ UQUPUQn[4        RP                  (       aH  URS                  5       S-  nURU                  5        SU SURW                  U5      RY                  5        3nU(       a  UsSSS5        $ U R[                  UUU5      Ul.        UsSSS5        $ ! , (       d  f       GM»  = f! , (       d  f       GNï= f! , (       d  f       GM:  = f! , (       d  f       GNÿ= f! , (       d  f       GN˜= f! , (       d  f       g= f)z;
Helper method to codegen a single template kernel variant
r>   r   Nz<LOAD_INPUT_rf  c              3  ó@   #   • U  H  oR                  5       v •  M     g 7frX   )Úcan_codegen_without_upcasts)rü   Úp_ns     r^   rý   Ú:SIMDScheduling._codegen_single_template.<locals>.<genexpr>+  s   é € ð 5ÚES¸c×7Ñ7×9Ð9Â^ùó   ‚ztriton.codegen_upcast_to_fp32z<DEF_KERNEL>z	<ARGDEFS>F)Ústrictg    eÍÍAr¦  )/r  r  rµ   r  r²   ÚiterÚprologue_fused_inputsrÓ   rc  r   r�  rn  Úrangerh  Úset_subgraph_bodyrè   r2  rö  ÚcseÚ
invalidater   Únamed_input_nodesr{   rÖ   r-  r  r   rS  Ú#prologue_fused_inputs_preserve_zeror=   rQ  rÿ   r   r   r'   Úcurrent_originsr¨   ÚoriginsÚfinalize_hookÚfinalize_remainingrK  r¡  Úimports_for_benchmark_kernelÚcodegen_kernel_benchmarkÚgetvaluer�  r0  )rp   rn   ra  r\  r]  r^  rb  r_  Útemplate_readsÚprologue_groupÚprologueÚnamesÚresultr¨   Úpartial_codeÚnum_store_subgraphsrg  Úsubgraph_nameÚ
input_nameÚbufferÚcan_codegen_without_upcastÚprologue_noder\  r&  Únum_gbs                            r^   Ú_codegen_single_templateÚ'SIMDScheduling._codegen_single_templateê  s¡  € ð &(Ð"Ø&×8Ñ8Ó:ˆØˆÛ&ˆHØ×-Ñ-Ó/ˆEØ×!Ñ! (Ô+à×%Ñ%Ü˜5“z Q“Ð&�Ø@N¬4´°U³Ó+<Ñ=Ø×,Ñ,×0Ñ0´´d¸5³kÓ1BÔCØ!#’ñ 'ô �>Ó" aÓ'Ð'Ð'ð ×1Ñ1ØØØØØ&Ü(ØØó	
ˆð ÑØˆMâÞ$ð +Ð<¨^Ó<�DØ—M‘M–Oñ =ñ "›8ˆLà"(×"?Ñ"?Ó"AÐÜÐ.Ö/�Ø &× FÑ FÀqÓ I�Ø×-Ñ-¨mÕ<Û .˜ØŸ™ V×%@Ñ%@ÀÇÁÓARÓ%SÖTñ !/à—J‘J×)Ñ)¬*«,Ô7÷ =Ñ<ñ 0ð '-×&>Ñ&>×&DÑ&D×&FÑ"�
˜FØ".¨z¨l¸!Ð <�Ø%?×%CÑ%CØ—O‘OÓ% ró&ð �>÷ ô 25ñ 5ÙESó5ó 2Ð.ô  ŸšØ7Ð=WÔ9Wõð $×5Ñ5°mÕDÛ1? ä$'¨×(FÑ(FÓ(HÓ$IÈQÓ$NÜ(+¨NÓ(;¸qÓ(@ä'CÀM×'RÑ'RØ(.×(RÒ(RØ,9×,JÑ,JÓ,Lñ)*Õ(Rð !.× 5Ñ 5Ø$*×$?Ñ$?Ø(5×(@Ñ(@Ó(Bó%&ö!"ñ 2@ð #ŸJ™J×1Ñ1´*³,Ô?÷! E÷ò ñ 'G÷# ôp ×!Ò! &Õ)Ü˜l¬C×0Ñ0ä—Y‘Y×.Ñ.¨}×/AÑ/A×/IÑ/IÕJØ ×.Ñ.¨~Ô>÷ Kà×*Ñ*¨;¸uÐ*ÑEð %×6Ô6�
Ø".¨z¨l¸!Ð <�à×*Ñ*¨=ÀÐ*ÓGñ 7ð
 #)×"?Ñ"?Ó"AÐÜÐ.Ö/�Ø &× FÑ FÀqÓ I�à×*Ñ*¨=Ö9ñ 0ô
 ˜,¬×,Ñ,Ø'‘ð (×:Ñ:Ó<�àM˜nÐM¨mÐM¸nÐMˆMä×&×&Ø×9Ñ9Ó;¸cÑA�à×:Ñ:Ó<Ð=¸RØ�j Ø×6Ñ6°vÓ>×GÑGÓIÐJðLð ö !Ø÷M *Ñ)ðP "&×!3Ñ!3°H¸mÈVÓ!TˆFÔà÷U *Ñ)÷Y =×<ú÷& EÖDú÷÷ ú÷9 ŽVú÷v KÖJú÷ *Õ)úsŠ   ÃA+S=Ä?ASÆAS=Ç6.S=È$S*È6B:S	Ë0S*Ë8S=Ì#A	T!Í,TÍ>DT!Ò"T!Ó
SÓ	S=Ó
S'Ó"S*Ó*
S:Ó4	S=Ó=
TÔ
T	ÔT!Ô!
T/c                ó(  ^^• SSK Jm  U4S jm/ n[        UR                  5      U/-    H[  n[	        U[        [
        45      (       a&  UR                  [        U4S jU 5       5      5        MD  UR                  T" U5      5        M]     [        U5      $ )Nr   r&   c                óÒ   >• [        U T5      (       d  g [        U [        R                  5      (       a  U R                  5       n U R	                  5       =nc  g [        S U 5       5      $ )Nc              3  ó$   #   • U  H  ov •  M     g 7frX   rË   rû   s     r^   rý   ÚKSIMDScheduling._get_multikernel_shapes.<locals>.get_size.<locals>.<genexpr>…  s   é € Ð)¢D˜qœ¢Dùs   ‚)rÿ   r   ÚBaseViewÚunwrap_viewÚmaybe_get_sizerˆ  )r  r  r'   s     €r^   Úget_sizeÚ8SIMDScheduling._get_multikernel_shapes.<locals>.get_size~  sX   ø€ Ü˜c 6×*Ñ*ØÜ˜#œrŸ{™{×+Ñ+Ø—o‘oÓ'�Ø×*Ñ*Ó,Ð,�Ñ5ØÜÑ)¡DÓ)Ó)Ð)r`   c              3  ó4   >#   • U  H  nT" U5      v •  M     g 7frX   rË   )rü   Ú_argr  s     €r^   rý   Ú9SIMDScheduling._get_multikernel_shapes.<locals>.<genexpr>Š  s   øé € Ð @ºC°D¡¨$§ ºCùs   ƒ)r   r'   r%  Úinputsrÿ   rˆ  rµ   )rp   r¨   rÏ  r  r'   r  s       @@r^   Ú_get_multikernel_shapesÚ&SIMDScheduling._get_multikernel_shapesy  sr   ù€ õ 	 õ	*ð ˆÜ˜Ÿ™Ó$¨ vÔ-ˆCÜ˜#¤¤e˜}×-Ñ-Ø—
‘
œ5Ô @¹CÓ @Ó@ÖAà—
‘
™8 C›=Ö)ñ	 .ô
 �S‹zÐr`   c                óH   • U R                  U5      n[        S U 5       5      $ )Nc              3  óF   #   • U  H  n[        S  U 5       5      v •  M     g7f)c              3  ó¢   #   • U  HE  n[        U[        R                  5      =(       a    [        U[        R                  5      (       + v •  MG     g 7frX   ©rÿ   r�   ÚExprr  rû   s     r^   rý   ÚFSIMDScheduling._kernel_has_dynamic_shapes.<locals>.<genexpr>.<genexpr>’  s8   é € ð â�Aô ˜1œeŸj™jÓ)×N´*¸QÄÇÁÓ2NÔ.NÔNÚùs   ‚AAN)rE  )rü   Úshapes     r^   rý   Ú<SIMDScheduling._kernel_has_dynamic_shapes.<locals>.<genexpr>‘  s2   é € ð 
ò
  �ô	 ñ áó÷ ð ò  ùs   ‚!)r"  rE  )rp   r¨   Úshapess      r^   Ú_kernel_has_dynamic_shapesÚ)SIMDScheduling._kernel_has_dynamic_shapes�  s.   € Ø×-Ñ-¨dÓ3ˆÜñ 
ñ
  ó
ó 
ð 	
r`   c                óP   ^• U R                  U5      n[        U4S jU 5       5      $ )z[
Returns cache key for hint-based multi-graph; key is tuple of shapes with hint filled in.
c              3  óN   >#   • U  H  n[        U4S  jU 5       5      v •  M     g7f)c              3  ó¨   >#   • U  HG  n[        U[        R                  5      (       a!  [        U[        R                  5      (       d  TOUv •  MI     g 7frX   r'  )rü   rØ   Úhints     €r^   rý   ÚASIMDScheduling._make_shape_cache_key.<locals>.<genexpr>.<genexpr>¡  sG   øé € ð ò �Aô ˜a¤§¡×,Ñ,´ZÀÄ5Ç=Á=×5QÑ5Qñ àôò ùs   ƒAANrï  )rü   r*  r2  s     €r^   rý   Ú7SIMDScheduling._make_shape_cache_key.<locals>.<genexpr>   s5   øé € ð 
ò  �ô ô ñ ó	÷ ð ò  ùs   ƒ"%)r"  rˆ  )rp   r¨   r2  r,  s     ` r^   Ú_make_shape_cache_keyÚ$SIMDScheduling._make_shape_cache_key™  s1   ø€ ð ×-Ñ-¨dÓ3ˆÜô 
ñ  ó
ó 
ð 	
r`   ©rb  Úhint_overridec          
     óÎ  • UR                   u  nu  pxUS:X  d   e[        UR                  [        5      (       Ga#  UR                  R                  (       Ga  [        UR                  R                  5      S:”  Gaã  U R                  UR                  5      (       GaÂ  0 n	/ n
UR                  R                  R                  5        Hœ  u  nnU" UR                  US9u  pÞU(       a>  U R                  UUUUUSS9n[        U[        5      (       d   eU
R                  U5        M^  Uc  Mc  U R                  UUUUUSS9nUc  SOU R                  UR                  U5      nXÙU'   Mž     U(       a  SR                  U
5      $ [        R                  " [        U	R!                  5       5      5        [#        U	5      n/ UQUPUQnU R%                  UUR&                  5        UR)                  UR&                  5        [*        R,                  =R.                  UR.                  -  sl        [*        R,                  =R0                  UR0                  -  sl        U R3                  5         gUR                  R5                  UR                  US9u  pÞU(       a  U R                  UUUUUSS9$ U R                  UUUUUSS9n/ UQUPUQnU R%                  UUR&                  5        UR)                  UR&                  UR                  5        [*        R,                  =R.                  UR.                  -  sl        [*        R,                  =R0                  UR0                  -  sl        U R3                  5         g)z¦
Codegen a triton template with multi-kernel dispatch support

If `only_gen_src_code=True` the src code will be returned instead of being
codegenned into the wrapper
r>   )r8  Trï  NFz

)r  rÿ   r¨   r   Ú_make_kernel_rendersr  r-  r{   r  r   rµ   r5  rÁ  rD   rÐ  r%  r¦   rE   r‘  r0  rt  r=   r®   rŒ  r’  r—  Úmake_kernel_render)rp   r\  r]  r^  rb  r8  r  Ú_numelr)  rÜ  Ú	src_codesr  r;  rn   ra  r\  Úshape_cache_keyÚmulti_kernelr&  s                      r^   Úcodegen_templateÚSIMDScheduling.codegen_templateª  sD  € ð  ,×1Ñ1ÑˆÑˆFØ˜‹{Ðˆ{ô �}×)Ñ)Ô+>×?Ò?Ø×"Ñ"×7×7Ð7Ü�M×&Ñ&×;Ñ;Ó<¸qÔ@Ø×/Ñ/°×0BÑ0B×CÒCàˆGØˆIð
 ×#Ñ#×8Ñ8×>Ñ>Ö@ñØØ"á!3Ø!×&Ñ&°mñ"‘�ö %Ø#×<Ñ<ØØØ%Ø&Ø&Ø*.ð  =ð  �Hô & h´×4Ñ4Ð4Ð4à×$Ñ$ XÖ.à Ñ(Ù Ø!×:Ñ:ØØØ%Ø&Ø&Ø*/ð ;ð �Fð %Ñ,ñ à!×7Ñ7¸×8JÑ8JÈIÓVð $ð 06˜OÓ,ñE AöH !Ø—{‘{ 9Ó-Ð-ä×0Ò0´°g·n±nÓ6FÓ1GÔHÜ.¨wÓ7ˆLØM˜nÐM¨mÐM¸nÐMˆMØ× Ñ  °×0HÑ0HÔIØ×$Ñ$ \×%=Ñ%=Ô>Ü�G‰G×#Ò# |×'CÑ'CÑCÕ#Ü�G‰G×&Ò&¨,×*IÑ*IÑIÕ&Ø×*Ñ*Ô,Øà*×/Ñ/×BÑBØ×"Ñ"°-ð Cð ‰NˆFö !Ø×4Ñ4ØØØ!Ø"Ø"Ø&*ð 5ð ð ð ×6Ñ6ØØØ!Ø"Ø"Ø&+ð 7ð �ð !R .Ð Q°-Ð QÀ.Ð Q�Ø×$Ñ$ ]°F×4FÑ4FÔGØ×"Ñ" 6×#5Ñ#5°}×7IÑ7IÔJä—‘×'Ò'¨6×+AÑ+AÑAÕ'Ü—‘×*Ò*¨f×.GÑ.GÑGÕ*Ø×.Ñ.Ô0Ør`   c                ó¢   • [         R                  R                  R                  [         R                  R                  R                  5       5        g rX   )r=   r®   rk  r”  Ú
device_opsÚsynchronizert   s    r^   Úcodegen_syncÚSIMDScheduling.codegen_sync	  s-   € Ü	�‰×Ñ×&Ñ&¤q§w¡w×'9Ñ'9×'EÑ'EÓ'GÕHr`   c           
     ó¦  • SSK Jn  SSKJn  [	        U R
                  U5      (       d   eU Vs/ s H  oˆR                  5       PM     n	n0 n
[        X5       H”  u  p¼[        US S9R                  u  nu  pïU R                  XÎU5      nU R                  UXï5      n[        UXï5      nUR                  5       =(       a    [        R                  R!                  USS9n[#        UUUUUUS9X«'   M–     UR%                  UU UU
S	9n[&        R)                  S
[+        U5      U Vs/ s H  n[+        U5      PM     sn5        / nU GHs  n[+        U5      S:X  a  M  [+        U5      S:X  a«  U
US      nU(       a  UR-                  SSU45        MI  U R                  UR.                  UR0                  S9nU R3                  UUR4                  U5        [        R6                  " U5         UR9                  5       nSSS5        UR-                  WUU45        MÏ  U" U R
                  UUS9nU Hh  nX«   nUR;                  UR.                  UR0                  U(       + U R
                  S9nU R3                  UR=                  U5      UR4                  U5        Mj     UR9                  5       nUR-                  UUU45        GMv     U$ s  snf s  snf ! , (       d  f       NÙ= f)a<  
Generate kernel code for combo kernel partitions.

Partitions subkernel_nodes using horizontal_partition(), then generates
kernel code for each partition. Single-node partitions are generated as
regular kernels, while multi-node partitions use ComboKernel.

Returns a list of (src_code, kernel, node_group) tuples.
r>   )ÚTritonKernel)ÚComboKernelc                ó4   • [        U R                  5       5      $ rX   r¬  r­  s    r^   rÌ   Ú;SIMDScheduling.generate_combo_kernel_code.<locals>.<lambda>7	  ó   € ¼#¸a¿n¹nÓ>NÔ:Or`   rÎ   F)rE  )r&  r(  rl   r)  r*  r+  )rš   Útriton_schedulingÚcustom_algorithmÚnode_info_mapz1ComboKernels: %d nodes partitioned into %s groupsr   N©r*  )Útriton_kernel_clsÚenable_autotuneÚmixed_sizes)r*  Úoptimize_maskrQ  )r[   rH  Útriton_combo_kernelrI  Ú
issubclassrë  r  râ  rÎ  r  r>  r  rJ   ru   r=   ÚchoicesrG  r$  Úhorizontal_partitionr­  r°  r  rµ   r(  r*  rÊ  r&  rQ  rÕ  Úcreate_triton_kernelÚcreate_sub_kernel)rp   Úsubkernel_nodesÚcustom_part_algorithmrR  rS  rb  rH  rI  r¨   Úfused_node_listsÚnode_schedule_mapÚpnrš   r  rl   r)  r&  r(  r*  r+  Ú
partitionsÚpÚkernel_code_listÚ
node_groupÚ	node_inforn   r\  Ú	subkernels                               r^   Úgenerate_combo_kernel_codeÚ)SIMDScheduling.generate_combo_kernel_code	  sî  € õ" 	)Ý4ô ˜$×*Ñ*¨L×9Ñ9Ð9Ð9á9HÓIº°ŸN™NÖ,¹ÐÐIØ13ÐÜ˜_Ö?‰IˆBÜ!$ UÑ0OÑ!P×!VÑ!VÑˆA‰�Ø ×7Ñ7¸ÀfÓMˆMØ×'Ñ'¨°uÓEˆFÜ)¨-¸ÓGˆHà×%Ñ%Ó'÷ Ü—I‘I×=Ñ=Ø°Eð >ð ð $ô %-Ø+ØØØØ!Ø(?ñ%ÐÓ!ñ @ð( !×5Ñ5Ø!Ø"Ø2Ø+ð	 6ð 
ˆ
ô 	�	‰	Ø?Ü�Ó Ù'Ó(šZ˜ŒS�ŽV™ZÑ(ô	
ð
 ÐÜ$ˆJÜ�:‹ !Ó#Ùä�:‹ !Ó#à-¨j¸©mÑ<�	Þ$à$×+Ñ+¨T°4¸Ð,DÖEð "×-Ñ-Ø!×(Ñ(Ø!*×!3Ñ!3ð .ð �Fð ×'Ñ'Ø 	× 7Ñ 7Ð9Jôô ×-Ò-¨fÕ5Ø#)×#8Ñ#8Ó#:˜÷ 6ð %×+Ñ+¨X°v¸zÐ,JÖKñ %Ø&*×&6Ñ&6Ø$3Ø +ñ�ó
 %�BØ 1Ñ 5�IØ +× @Ñ @Ø!×(Ñ(Ø!*×!3Ñ!3Ø*5¤oØ*.×*:Ñ*:ð	 !Að !�Ið ×'Ñ'Ø×0Ñ0°Ó;Ø!×/Ñ/Ø)öñ %ð "×0Ñ0Ó2�à ×'Ñ'¨°6¸:Ð(F×Gñ] %ð`  Ðùòe Jùò> )÷, 6Õ5ús   ®J8ÄJ=ÇKË
K	c                óÈ  • UR                  5       nUR                  nUR                  n[        R                  S:„  =(       d    [        R                  S:H  =(       a    UnU R                  X#XE5      nU H\  u  pxn	U R                  Xq/U5      n
U R                  UR                  U
5        [        R                  SU
5        UR                  U
5        M^     U R                  5         g )Nr>   z"ComboKernels: generated kernel %s.)Úget_subkernel_nodesÚuse_custom_partition_algorR  r   Úcombo_kernel_allow_mixed_sizesrf  r�  r‘  Úsnodesr­  r°  rt  r—  )rp   Úcombo_kernel_noder[  r\  rR  rS  rb  r\  rn   r  r0  s              r^   Úcodegen_combo_kernelÚ#SIMDScheduling.codegen_combo_kernelˆ	  sÓ   € Ø+×?Ñ?ÓAˆØ 1× KÑ KÐØ+×;Ñ;ˆÜ×;Ñ;¸aÑ?÷ 
Ü×1Ñ1°QÑ6×PÐ;Pð 	ð  ×:Ñ:Ø°Oó
Ðó $4ÑˆH˜aØ×,Ñ,¨XÐ7JÈFÓSˆKØ× Ñ Ð!2×!9Ñ!9¸;ÔGÜ�I‰IÐ:¸KÔHØ×Ñ˜{Ö+ñ	 $4ð 	×&Ñ&Õ(r`   é    c           
     ó¾  ^ ^^
• TS:H  nSU UU
4S jjnUR                  5       u  nm
[        U5      S::  a  [        T
5      S::  d  [        UT
-   5      (       a  / $ UR                  5       u  nm
U" UU(       a  UOT
UR                  U5      5      nU Vs/ s H=  n[	        T R                  UR                  UT5      UR                  UR                  S9PM?     n	nU	$ s  snf )Nr>   c                óº  >• [        UR                  5      [        U5      :X  d   SUR                  < SU< 35       eUR                  UR                  /n[	        S [
        R                  R                  U5       5       5      (       d   e[
        R                  R                  U5       Vs/ s HF  nUR                  [        R                  R                  ;  d  M-  [        U[        5      (       d  MD  UPMH     nn[        UR                   Vs/ s H  oDR                  PM     sn5      nSS jn[        TR!                  U" U5      /U 5      SSS9/nU GHæ  n[        R                  R"                  R%                  UR&                  UR                  5      n	[        U	5      [        U5      :X  d   e U	R'                  S5      S-   n
U
[        U5      :X  a  M‚  [	        S	 XšS
  5       5      (       a  M�   U" US
U
 5      U" XS
 5      4n[        R                  R"                  R+                  [-        S [/        X5       5       5      5      nUR                  U;   a  US-  n[        R1                  US   5      (       a  US-  n[        R1                  US   5      (       a  US-  n[        R                  R"                  R+                  U[-        [
        R                  " UT5      5      -
  5      S:¼  d  GMŸ  UR3                  [        TR!                  U" US
U
 5      U" XS
 5      /T5      UUR                  S95        GMé     U$ s  snf s  snf ! [(         a     GM  f = f)z@
Compute tiling candidates by dividing up the iteration ranges.
zrw.range_vars=z ranges=c              3  óN   #   • U  H  n[        U[        [        45      v •  M     g 7frX   )rÿ   r"   r#   )rü   rŸ  s     r^   rý   ÚHSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.<genexpr>¬	  s&   é € ð âE�Cô ˜3¤¬GÐ 4×5Ð5ÚEùs   ‚#%c                óf   • [         R                  R                  R                  [	        U 5      5      $ rX   r6  )rô  s    r^   Úcollapse_rangesÚNSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.collapse_ranges¸	  s"   € Ü—w‘w×'Ñ'×0Ñ0´¸vÓ1FÓGÐGr`   Únoner   )r(  ri   Úscorer>   c              3  ó*   #   • U  H	  oS :H  v •  M     g7frù  rË   rû   s     r^   rý   rt  Î	  s   é € Ð;ª? a ž6ª?ùs   ‚Nc              3  ó:   #   • U  H  u  pUS :w  d  M  Uv •  M     g7frù  rË   )rü   r  r¬  s      r^   rý   rt  Ý	  s   é € ð "Ú1E¡ ÈÐSTÉŸ™Ò1Eùs   ‚’	r   ©r(  ry  ri   )rô  rá  r…   r‚   )r  Ú
range_varsr$  r.  r  rA  rµ  r¶  ri   r=   r®   rŒ  rÿ   r"   r   ÚCandidateTilingÚcreate_partial_tilingr¯   Ústride_hintsr™   Ú
ValueErrorr  r8   râ  Úis_good_sizerµ   )Úis_pointwiserô  ÚrwÚdep_sourcesrŸ  ÚdepsÚwrite_namesrv  Útilingsrƒ  ÚsplitÚtiled_groupsry  r'  r(  Úreduction_rangess                €€€r^   Útile_rangesÚ5SIMDScheduling.candidate_tilings.<locals>.tile_ranges¡	  s#  ø€ ô �r—}‘}Ó%¬¨V«Ó4ÐS¸¸¿¹Ñ8HÈ	È&ÉÐ6SÓSÐ4ð Ÿ8™8 R§Y¡YÐ/ˆKÜñ ä$Ÿ?™?×8Ñ8¸ÔEó÷ ñ ð ð ô %Ÿ?™?×8Ñ8¸ÔEóâE�CØ—8‘8¤1§7¡7×#:Ñ#:Ñ:ó ô ˜s¤I×.÷ ÙEð ð ô %¸"¿)º)Ó%Dº)°3§h¤h¹)Ñ%DÓEˆKôHô
  Ø×4Ñ4Ù(¨Ó0Ð1°<óð  ØñðˆGô �ÜŸ'™'×*Ñ*×7Ñ7¸¿	¹	À2Ç=Á=ÓQ�Ü˜7“|¤s¨6£{Ó2Ð2Ð2ð
Ø#ŸM™M¨!Ó,¨qÑ0�EØ¤ F£Ó+Ù ÜÑ;¨7°6©?Ó;×;Ñ;ñ !ð <ñ $ F¨6¨E NÓ3Ù# F¨6 NÓ3ð �ô Ÿ™×(Ñ(×2Ñ2Ü!ñ "Ü14°VÔ1Eó"ó ó�ð
 —8‘8˜{Ó*à˜Q‘J�EÜ"×/Ñ/°¸Q±×@Ñ@Ø˜Q‘J�EÜ"×/Ñ/°¸Q±×@Ñ@Ø˜Q‘J�Eô —G‘G×$Ñ$×.Ñ.Ø¤¬i¯oªo¸fÐFVÓ.WÓ XÑXóð öð
 —N‘NÜ'Ø#&×#<Ñ#<á$3°F¸6¸E°NÓ$CÙ$3°F¸6°NÓ$Cð!"ð !0ó$ð #(Ø!$§¡ñ
÷ñQ ðl ˆNùò[ùò &Eøô: "ó Ûðús0   Â,,MÃMÃ3MÄMÆ1#MÇMÍ
MÍMr|  )rƒ  rˆ   r…   úlist[CandidateTiling])	rö  r  r   Ú"pointwise_or_reduction_read_writesr~  Úcomplete_partial_tilingr(  ry  ri   )r'  r¨   rl   r(  rƒ  rŒ  Úpointwise_rangesÚpartial_tilingsr(  Úfull_tilingsr‹  s   `  `      @r^   Úcandidate_tilingsÚ SIMDScheduling.candidate_tilingsœ	  sþ   ú€ ð '¨!Ñ+ˆ÷\	ñ \	ð| .2¯_©_Ó->Ñ*ÐÐ*äÐ Ó! QÓ&ÜÐ$Ó%¨Ó*Ü$Ð%5Ð8HÑ%H×IÑIàˆIð .2¯_©_Ó->Ñ*ÐÐ*Ù%ØÞ ,ÑÐ2BØ×3Ñ3°LÓAó
ˆñ *ó	
ò *�ô Ø×2Ñ2Ø—M‘M 5¨/óð —l‘lØ—[‘[ôñ *ð 	ð 	
ð Ðùò	
s   ÂACc                ó†   • / SQ[        U5      * S nSS/S[        U5       n[        / [        X15      Q[        XB5      Q5      $ )z;
Create a tiling dict from pointwise and reduction splits.
)rR   rS   rT   NrU   rV   )r  r   râ  )r'  Ú	pw_tilingÚreduction_tilingÚpw_prefixesÚreduction_prefixess        r^   Úcreate_tilingÚSIMDScheduling.create_tiling
  sT   € ò &¤s¨9£~ oÐ&7Ð8ˆØ# U˜^Ð,C¬cÐ2BÓ.CÐDÐÜØVŒc�+Ó)ÐV¬CÐ0BÓ,UÐVó
ð 	
r`   c                óR   • U R                  U(       a  UO/ U(       d  U5      $ / 5      $ rX   )r›  )r'  r(  rƒ  s      r^   r  Ú$SIMDScheduling.create_partial_tiling*
  s0   € ð × Ñ Þ"‰F¨Þ&ˆFó
ð 	
à,.ó
ð 	
r`   c                óœ   • [        UR                  5       5      nSU;   nX#-  nU[        U5      -  /nU(       a  XG4OXt4nU R                  " U6 $ )zR
Given a tiling for only pointwise or reduction dimensions, adds the missing one.
rT   )r%  r¦   r8   r›  )	r'  r(  rl   r(  Úsplitsrƒ  Útotal_numelÚmissing_tilingÚtiling_argss	            r^   r�  Ú&SIMDScheduling.complete_partial_tiling5
  s^   € ô �f—m‘m“oÓ&ˆØ˜f‘}ˆàÑ-ˆØ%¬°fÓ(=Ñ=Ð>ˆö )5ˆVÑ$¸>Ð:Rð 	ð × Ò  +Ð.Ð.r`   c           
     ó
  • US:H  n[         [        [        [        R                  4      " 5       n[
        R                  " U5       GH   n[        U[        R                  5      (       d  M%  UR                  5       nU(       d  [        US   5      S:X  a  MP  Xt(       a  SOS   nU/n	UR                  R                  5        V
s/ s H7  n
[        U
[        5      (       d  M  [        U
R                  5      S:”  d  M5  U
PM9     nn
U GHÜ  n
/ U
R                  R!                  5       Qn[        R"                  R$                  n[&        R(                  R*                  nSn[-        U5       H(  u  nu  nnUU-  nUnUR/                  XÒ5      (       d  M(    O   UR1                  XÒ5      (       d  M¥  US-   nU(       a  USU OUUS n/ nU H›  u  nn[2        R4                  " U
R6                  U5      n[9        SUR;                  [<        5      UR;                  [>        5      -   [        U5      5      n[2        R@                  " UUUU5      nUb  US   OU/nURC                  U5        M�     U Vs/ s HN  n[&        R(                  R*                  R1                  U[        R"                  R$                  5      (       a  ML  UPMP     nn[        U5      S:”  d  GMË  U	RE                  U5        GMß     U	 H{  n[9        S[        U5      [G        S5      -
  5      nUS-   n[I        USU 5      nU4[K        UUS 5      -   n URM                  U RO                  U RQ                  U U5      UU5      5        M}     GM#     [S        U[        SS9n!U!$ s  sn
f s  snf )z°
Creates N-dimensional tiling candidates, attempting to simplify loads/stores
by tiling the kernel into higher dimensions.

Returns a list of tilings ranked by dimensionality.
r>   r   Nr   T)rÏ   Úreverse)*r   r   r   r�   r(  rG   rŽ  rÿ   r   rO  rö  r  r#  Úreads_and_writesr"   rô  r{   r�   r‘   r=   r®   r¯   r‘  Ústatically_known_geqr°   r?   Úget_subexpr_involving_symbolr™   rÎ  rB  r   r   Úmatch_mod_div_block_exprr¢  rµ   r_   r8   rˆ  rÓ   r�  r  rc  )"r'  r&  Úpointwise_numelr(  rƒ  rˆ  r¨   Únode_rangesÚranges_to_tileÚnode_tilingsrŸ  Úmemory_depsÚall_var_rangesÚpointwise_vars_numelr¯   Úpointwise_end_idxr“  Ú_varrl   Úreduction_start_idxrk   Úindex_tilingÚvarr™   Únum_dimsÚmatch_resultÚdimsÚdimÚnode_tilingÚnum_leading_dimsÚfirst_trailing_dimÚcollapsed_leading_dimÚcollapsed_splitsÚranked_tilingss"                                     r^   Úget_nd_tilingsÚSIMDScheduling.get_nd_tilingsJ
  s  € ð '¨!Ñ+ˆÜœ^¬C´·±¨OÑ<Ò=Ó?ˆÜ#×*Ò*¨=×9ˆDÜ˜d¤I×$;Ñ$;×<Ñ<Ùð Ÿ/™/Ó+ˆKÞ¤C¨°A©Ó$7¸1Ó$<Ùð )­l©ÀÑBˆNØ*Ð+ˆLð  ×+Ñ+×<Ñ<Ô>óâ>�CÜ˜c¤9×-ó ä25°c·j±j³/ÀAÑ2E÷ Ù>ð ð ô
 #�ð "7 3§:¡:×#3Ñ#3Ó#5Ð!6�Ü',§w¡w§{¡{Ð$ÜŸ7™7×+Ñ+�Ø$%Ð!Ü*3°NÖ*CÑ&�C™˜$ Ø(¨EÑ1Ð(Ø(+Ð%Ø×4Ñ4Ø,÷ó ñ ñ +Dð  ×7Ñ7Ø(÷ñ ñ ð '8¸!Ñ&;Ð#ö $ð #Ð#7Ð$7Ñ8à'Ð(;Ð(<Ð=ð ð  "�Û",‘J�C˜Ü/×LÒLØŸ	™	 3ó�Eô
  #ØØŸ™¤HÓ-°·±¼OÓ0LÑLÜ˜NÓ+ó �Hô $7×#OÒ#OØ˜s E¨8ó$�Lð /;Ñ.F˜<¨š?ÈUÈG�DØ ×'Ñ'¨Ö-ñ% #-ñ.  ,ó â+˜ÜŸ7™7×+Ñ+×CÑCÀCÌÏÉÏÉ×U÷ Ù+ð ð  ô �|Ó$ qÖ(Ø ×'Ñ'¨×5ñw #ó|  ,�Ü#& q¬#¨kÓ*:¼]È1Ó=MÑ*MÓ#NÐ Ø%5¸Ñ%9Ð"Ü(5°kÐBUÐCUÐ6VÓ(WÐ%Ø$9Ð#;¼eØÐ 2Ð 3Ð4ó?ñ $Ð ð —‘Ø×/Ñ/Ø×1Ñ1Ð2BÀLÓQØ'Ø'óöô  ,ñi :ôJ  ØÜØñ
ˆð Ðùòuùòr s   ÃM;ÃM;Ã9M;É%AN Ê4N c                ó‚  ^^^^^^^^^^^^^• TR                   (       d  SOTR                   R                  mTR                  R                  mTR                  R                  mTR                  R
                  nT Vs/ s H  oeU   PM	     snmT Vs/ s H  oeU   PM	     snm[        R                  R                  R                  n[        R                  " U" [        T5      5      U" T5      :H  UUU4S j5        [        R                  " U" [        T5      5      U" T5      :H  UUU4S j5        0 m/ n   S       SUUUUUUUUU4	S jjjn	UR                  U	" SS9U	" SS945        T(       a  UR                  U	" T4SSS9U	" SS945        TTR                  R                  5       -  n
U
 H   nUR                  U	" U4SS9U	" SS945        M"     [!        S	S
9S	:X  a@  TS:X  a:  ["        R$                  " U
S5       H  nUR                  U	" USS9U	" SS945        M!     / nU H^  u  u  pÞu  nn['        U R)                  Xß5      [+        U5      [+        U5      -   S9nU R)                  UU5      nUR                  UU45        M`     U R)                  T/T/5      nSmSm[+        TR,                  R/                  5       5      mUUU4S jn[1        UUS9 HÛ  u  nnU R3                  TTTUR4                  5      (       d  UR4                  U:X  a‚  [7        UR4                  5      TS:X  a  SOS-
  nU[!        S	S
9:”  aE  [8        R;                  SU[        R<                  R>                  R@                  RB                  5        M«  UR4                  U4s  $ UR4                  U:X  d  MÍ  UR4                  U4s  $    US4$ s  snf s  snf )zb
Generates a tiling, and a score of each tile according to each tile's coalesced memory accesses.
Nc                 ó   >• T ST ST  3$ ©Nr¢   rË   )r&  r«  Ú	pw_rangess   €€€r^   rÌ   Ú8SIMDScheduling.compute_tiling_strategy.<locals>.<lambda>â
  s   ø€ �y�k  OÐ#4°B°}°oÑFr`   c                 ó   >• T ST ST  3$ rÅ  rË   )r&  Ú
red_rangesr(  s   €€€r^   rÌ   rÇ  ç
  s   ø€ �z�l " _Ð$5°R¸°ÑGr`   Fc                ó�  >	• U(       a  TOTnU(       a  TOTnU(       d  U(       a  U// 4$ / / 4$ [        U 5      X4nTR                  U5      =n(       a  U$ U(       a  TOTn/ n/ n	Sn
Sn[        Xs5       GH  u  pÍXÀ;  a"  X­-  n
TR                  R                  US5      nM-  U(       a©  UT:X  a£  TR                  nUc   eUR
                  n[        XÞR
                  5      nUR                  U
U-  5        U	R                  UR                  5        UR                  U5        U	R                  TR                  R                  US5      5        Sn
SnMÝ  X­-  n
UR                  U
5        U	R                  TR                  R                  US5      5        Sn
GM"     U
S:w  d  U(       a1  [        U5      S:X  a"  UR                  U
5        U	R                  U5        [        [        U5      5       HQ  n[        R                  R                  R                  UU   SS9n[        US5      n[!        U	U   U-  S-  5      U	U'   MS     X‰4TU'   X‰4$ )zE
Generate a tiling, and a tiling score, given vars to use as splits.
r>   r   rp  ©Úfallbackrg  )r  rÖ   râ  Úcoalesced_by_varÚsuggested_splitÚtiling_factorr   rµ   ry  r  rù  r=   r®   r¯   rÅ   rn  rÜ   )Úvars_to_useÚuse_split_varrƒ  rô  Útarget_numelrÏ   rÏ  Úsplitting_varsr   Úsplit_scoresÚprodÚprev_var_coalesced_scorer€  Úv_rangeÚ
var_tilingÚtileÚ	remainderrg  rØ   Úall_iter_varsÚall_red_varsr²  r«  rÆ  rÉ  r(  Úscored_sub_splitÚ
tiling_vars                      €€€€€€€€€r^   Úprocess_node_varsÚASIMDScheduling.compute_tiling_strategy.<locals>.process_node_varsñ
  s<  ø€ ö #/‘Y°JˆFÞ.:™?ÀˆLæÞØ)˜N¨BÐ/Ð/à ˜8�Oä˜Ó$ mÐBˆCØ&×*Ñ*¨3Ó/Ð/ˆsÕ/Ø�
æ.:™]ÀˆNàˆFØˆLØˆDØ'(Ð$ô " .×9‘
�ØÓ'Ø‘O�DØ/@×/QÑ/Q×/UÑ/UØ˜1ó0Ð,ñ æ  Q¨*£_Ø!2×!BÑ!B�JØ%Ñ1Ð1Ð1à%×3Ñ3�DÜ (¨×2JÑ2JÓ K�Ià—M‘M $¨Ñ"2Ô3Ø ×'Ñ'¨
×(8Ñ(8Ô9à—M‘M $Ô'Ø ×'Ñ'Ð(9×(JÑ(J×(NÑ(NÈqÐRSÓ(TÔUà�DØ/0Ð,áà‘�Ø—‘˜dÔ#Ø×#Ñ#Ð$5×$FÑ$F×$JÑ$JÈ1ÈaÓ$PÔQØ“ñ; :ð> �q‹yž\¬c°&«k¸QÓ.>Ø—‘˜dÔ#Ø×#Ñ#Ð$<Ô=ô œ3˜v›;Ö'�Ü—G‘G×$Ñ$×6Ñ6°v¸a±yÈ2Ð6ÐN�Ü˜˜1“I�Ü"% l°1¡o¸Ñ&9¸AÑ&=Ó">�˜Q“ñ (ð
 &,Ð$:Ð˜SÑ!ØÐ)Ð)r`   T)rƒ  )rÑ  rƒ  r   ©r]   r>   r   )ry  gffffffð?g®Gázð?c                óÒ   >• SnU S   R                   R                  5        H)  n[        R                  U5      (       d  UT-  nM$  UT-  nM+     TS-  nU S   R                  U-   * U-  $ )Ng      ð?r   gš™™™™™©?)r(  r¦   r~  r‚  ry  )r_  Úscore_factorÚ	tile_sizeÚuncoalesced_penaltyÚ"bad_size_additional_tiling_penaltyÚgood_size_tiling_penaltyÚtotal_uncoalesceds       €€€r^   Ú	score_modÚ9SIMDScheduling.compute_tiling_strategy.<locals>.score_modx  ss   ø€ ØˆLØ˜q™TŸ[™[×/Ñ/Ö1�	Ü&×3Ñ3°I×>Ñ>Ø#/Ð2TÑ#T’Là#/Ð2JÑ#J’Lñ	 2ð #4°dÑ":Ðà�q‘T—Z‘ZÐ"5Ñ5Ð6¸ÑEÐEr`   rÎ   r   zmFound optimal tiling with %s tiles but torch._inductor.config.triton.max_tiles set to %s. Consider increasing)rË   FF)rÐ  ztuple[sympy.Expr, ...]rÑ  rˆ   rƒ  rˆ   r…   ztuple[list[int], list[int]])"rÎ  r¶  Únorm_read_writesrÑ   Úreduce_varsrk   r=   r®   r¯   rÅ   rY   Ú_checkr8   rµ   rÍ  rì  r_   rA  Úcombinationsr~  r›  rt  Úuncoalesced_addrsr¦   rc  Útiling_is_compatibler(  r  Úperf_hint_logÚinforZ   r   r[   r\   ) r'  r&  r«  r(  r²  rô  r€  Úget_hintÚscore_splitrß  Úoverlapping_iter_varsrÐ  rˆ  Úpw_splitÚpw_scoreÚ	red_splitÚ	red_scoreÚ	candidaterÛ  Údefault_tilingré  ÚcandÚ
tiling_lenrÛ  rÜ  ræ  rç  rÆ  rÉ  rÝ  rÞ  rè  s     ````                  @@@@@@@@@r^   Úcompute_tiling_strategyÚ&SIMDScheduling.compute_tiling_strategyÆ
  sæ  ÿü€ ð %×4×4ñ à"×2Ñ2×6Ñ6ð 	ð *×:Ñ:×EÑEˆØ(×9Ñ9×EÑEˆØ"×3Ñ3×>Ñ>ˆá(5Ó6ª 1˜A”Y©Ñ6ˆ	Ù)5Ó6ª A˜Q”i©Ñ6ˆ
ô —7‘7×#Ñ#×5Ñ5ˆÜ�ŠÙ”] 9Ó-Ó.±(¸?Ó2KÑKÞFô	
ô
 	�ŠÙ”] :Ó.Ó/±8¸OÓ3LÑLÞGô	
ð DFÐð ð 	ð
 35Ø"'Ø!&ðK	*Ø/ðK	*àðK	*ð ðK	*ð )÷	K	*÷ K	*ð K	*ð\ 	×Ñá!¨tÑ4Ù!¨uÑ5ðô	
ö Ø×Ñá%Ø#˜°TÈññ &°5Ñ9ð	ôð Ð-×>Ñ>×CÑCÓEÑEð 	ó 'ˆAØ×Ñá% q d¸Ñ>Ù%°5Ñ9ðöñ 'ô  Ñ# qÓ(¨_ÀÓ-AÜ(×5Ò5Ð6KÈQÖO�Ø×"Ñ"á)¨+ÀDÑIÙ)°uÑ=ðöñ  Pð RTˆÛ<GÑ8Ñ ˆXÑ"8 9¨iÜ'Ø×!Ñ! (Ó6Ü˜(“m¤c¨)£nÑ4ñˆIð ×,Ñ,¨X°yÓAˆLØ�N‰N˜I |Ð4Ö5ñ =Hð ×*Ñ*¨OÐ+<¸Ð>OÓPˆð .3Ð*Ø#(Ð äÐ 1× CÑ C× JÑ JÓ LÓMÐ÷	Fô #)¨°iÔ"@ÑˆD�,à×(Ñ(Ø! ?°OÀTÇ[Á[÷ñ ð —;‘; .Ó0ô ! §¡Ó-°oÈÓ6J±ÐPQÑR�
Ø¤°aÑ 8Ó8Ü!×&Ñ&ð9à"ÜŸ™×.Ñ.×5Ñ5×?Ñ?ô	ñ à—{‘{ LÐ0Ò0ð �{‰{˜nÕ,Ø—{‘{ LÐ0Ò0ñ/ #Að2 ˜tÐ#Ð#ùòK 7ùÚ6s   Á=N7ÂN<c                ó`   ^^• [        T[        5      (       d   e[        UU4S jU 5       5      $ )Nc              3  óÈ   >#   • U  HW  n[        U[        R                  5      (       d  M$  [        R	                  TR                  5       UR                  5       TS 9v •  MY     g7frõ  )rÿ   r   rO  rƒ   r-  r¦   rö  )rü   r¨   r(  r(  s     €€r^   rý   Ú6SIMDScheduling.tiling_is_compatible.<locals>.<genexpr>«  sR   øé € ð 
ò &�Ü˜$¤	× 7Ñ 7×8ó	ŒJ×$Ñ$Ø—‘“ §¡Ó!2ÀOð %õ ò &ùs
   ƒ#A"ª8A")rÿ   r'  r  )r'  r&  rl   r(  r(  s      ``r^   rð  Ú#SIMDScheduling.tiling_is_compatible¢  s4   ù€ ô ˜&¤$×'Ñ'Ð'Ð'Üõ 
ñ &ó	
ó 
ð 	
r`   c                óL   • U H  nU R                  XX55      (       d  M  Us  $    g rX   )rð  )r'  r&  rl   r(  rÀ  r(  s         r^   Úget_first_compatible_tilingÚ*SIMDScheduling.get_first_compatible_tiling³  s+   € ó %ˆFØ×'Ñ'¨¸o×VÓVØ’ñ %ð r`   c                ó,   • U R                  XX45      S   $ rÖ  )rÏ  )r'  r&  rl   r(  r²  s        r^   r  ÚSIMDScheduling.select_tilingÁ  s$   € ð ×(Ñ(Ø /ó
à
ñð 	r`   c                ó”  • US:H  nU R                  U/U/5      n[        R                  " U5       Hž  n[        UR                  [
        R                  5      (       d  M.  UR                  R                  5       S:X  d  MN  [        R                  R                  (       d  Mo  UR                  5       nUS   n	US   n
U R                  Xš5      nUS4s  $    [        R                  R                  R                  R                  (       a8  U(       a1  [        R                  R                  (       d  U R!                  XX45      $ U(       d  [        R                  R"                  (       a  [%        SS9S::  a±  [&        R(                  [*        R,                  ::  a‹  [        R                  " U5       Hq  n[        R                  R"                  (       a  M$  [/        U R1                  XrU5      5      S:”  d  ME  [&        R3                  [4        R6                  " S5      5          US4$    US4$ [9        5       n[:        R<                  " 5       n[        R                  " U5       Hl  nU R1                  XrU5       HS  nUR>                  U;   a  M  UR>                  b  URA                  UR>                  5        XÞ==   URB                  -  ss'   MU     Mn     URE                  5        VVs/ s H  u  pïURF                  PM     nnn[%        SS9S:¼  aH  U(       aA        SS	 jn[I        S[/        U5      5       H  nU" US   UU   5      nUc  M  U/U-   n  O   [/        U5      S:”  a  [&        R3                  S
U5        [        R                  R                  (       a  U RK                  XU5      U-   nU RM                  XUU5      =n(       a  US4$ US4$ s  snnf )z«
Heuristics to decide how to tile kernels.
Currently, we tile based on stride-1 dimensions.

Returns:
    `(tile1, tile2, reduction_numel)` s.t. `tile1 * tile2 == numel`

r>   r2  r   Nr   rá  z½
                                Reduction over non-contiguous dims.
                                Consider setting config.triton.tile_reductions to True.
                                r   c                ó:  • U S   U R                  SS5      p2US   UR                  SS5      pT[        X5/5      (       d/  [        R                  R                  R                  X5-
  5      S:X  a  g [        R                  R                  R                  X5-
  5      S:  a  XE4X#4su  p#u  pE[        R                  R                  R                  X5-
  5      S:”  d   e[        R                  R                  R                  X55      (       d  g U[        X55      UU S   S.nU$ )NrT   rS   r>   r   rU   )rR   rS   rT   rU   )rÖ   r   r=   r®   r¯   r  rý  r   )Útiling0r  Úa0Úa1Úb0Úb1Ú
new_tilings          r^   Úconvert_tiling_to_3dÚBSIMDScheduling.get_tiling_and_scores.<locals>.convert_tiling_to_3d)  sý   € ð ! ™ w§{¡{°3¸Ó':�BØ  ™ w§{¡{°3¸Ó':�Bô *¨2¨(×3Ñ3Ü—w‘w×'Ñ'×1Ñ1°"±'Ó:¸aÓ?àÜ—7‘7×#Ñ#×-Ñ-¨b©gÓ6¸Ó:à*,¨°B°8Ð&‘H�R™h˜rä—w‘w×'Ñ'×1Ñ1°"±'Ó:¸QÓ>Ð>Ð>Ü—w‘w×'Ñ'×DÑDÀR×LÑLØð Ü! "Ó)ØØ" 5™>ñ	�
ð "Ð!r`   zpossibly bad tiling: %s)r  rÜ  r  rÜ  r…   rÞ  )'r›  rG   rŽ  rÿ   r¨   r   rP  rQ  r   r[   rN  rö  rY   rZ   rµ  Úprefer_nd_tilingrþ  Útile_reductionsr_   rñ  ÚlevelÚloggingÚWARNINGr  r”  rò  ÚtextwrapÚdedentr   Úcollectionsr   ri   rÓ   ry  Úmost_commonr(  rù  rÁ  r  )r'  r&  rl   r(  r²  rƒ  rû  r¨   r¬  Ú	range_y_xÚrange_rr(  Ú
seen_namesÚcandidate_tilesÚcandidate_tilingry  rÀ  r  rg  Únew_3d_tilings                       r^   rÏ  Ú$SIMDScheduling.get_tiling_and_scoresÍ  sl  € ð" '¨!Ñ+ˆð ×*Ñ*¨E¨7°_Ð4EÓFˆô $×*Ò*¨=Ö9ˆDÜ˜$Ÿ)™)¤R×%6Ñ%6×7Ó7à—I‘I×0Ñ0Ó2°eÕ;ÜŸ™×3×3Ñ3ð #'§/¡/Ó"3�KØ +¨A¡�IØ)¨!™n�GØ ×.Ñ.¨yÓB�FØ! 4˜<Ò'ñ :ô  �O‰O×"Ñ"×)Ñ)×B×BÞ!Ü—M‘M×2×2à×.Ñ.Ø oóð ö ¤V§]¡]×%B×%BÄ}ØñH
àóHô ×"Ñ"¤g§o¡oÓ5Ü+×2Ò2°=ÖA�Dä"ŸM™M×9×9Ñ9Ü × 5Ñ 5°dÀ?Ó SÓTÐWXÕXä%×*Ñ*Ü$ŸOšOð!$óôð à! 4Ð'Ð'ñ Bð " 4Ð'Ð'ä&0£lˆ
Ü4?×4GÒ4GÓ4IˆÜ#×*Ò*¨=Ö9ˆDØ$'×$9Ñ$9¸$ÀÖ$WÐ Ø#×(Ñ(¨JÓ6ÙØ%×*Ñ*Ñ6Ø—N‘NÐ#3×#8Ñ#8Ô9ØÓ1Ð5E×5KÑ5KÑKÕ1ó %Xñ :ð ,;×+FÑ+FÔ+Hô7
â+HÑ'Ð ð ×#Ô#Ù+Hð 	ñ 7
ô
  Ñ# qÓ(®\ð"Ø.ð"Ø9Nð"à0ô"ô8 ˜1œc .Ó1Ö2�Ù 4Ø" 1Ñ% ~°aÑ'8ó!�ð !Ó,Ø&3 _°~Ñ%E�NÙñ 3ô ˆ~Ó Ó"Ü×ÑÐ8¸.ÔIô �=‰=×)×)à×"Ñ" =¸ÓIØ ñ!ð ð
 ×4Ñ4Ø /°>ó
ð 
ˆ6õ 
ð ˜4�<Ðà˜tÐ#Ð#ùóA7
s   ËOc                ó   • g rX   rË   rt   s    r^   ÚflushÚSIMDScheduling.flush_  rÙ  r`   c                ó   • gr†  rË   rt   s    r^   Úready_to_flushÚSIMDScheduling.ready_to_flushb  rˆ  r`   c           	     óò  • [        S U 5       5      (       d¹  [        US S9R                  u  nu  pVU R                  XU5      nU R	                  XuU5      nU R                  U[        XuU5      S9n	U R                  Xy5        [        R                  " SU5         [        R                  " U	5         U	R                  5       n
S S S 5        S S S 5        OJUS   R                  U5      u  p¼n[        R                  " SU5         U R                  UUUSUS9n
S S S 5        W
R                  [!        ["        R$                  5      S	5      n
U
$ ! , (       d  f       NŒ= f! , (       d  f       NJ= f! , (       d  f       N[= f)
Nc              3  ó@   #   • U  H  oR                  5       v •  M     g 7frX   )r  )rü   rÙ   s     r^   rý   ÚASIMDScheduling.generate_kernel_code_from_nodes.<locals>.<genexpr>h  s   é € Ð2ªE q—=‘=—?�?ªEùrõ  c                ó4   • [        U R                  5       5      $ rX   r¬  r­  s    r^   rÌ   Ú@SIMDScheduling.generate_kernel_code_from_nodes.<locals>.<lambda>i  rL  r`   rÎ   rP  rK  r   Tr7  rL  )rE  rÎ  r  r>  r  rë  rJ   rN  r   rS  r=   rQ  rÕ  Úget_prologue_template_epiloguer@  rT  r   r5   rU  )rp   rš   rK  r8  r  rl   r)  r&  r(  rn   r\  r  ÚtemplateÚepilogues                 r^   Úgenerate_kernel_code_from_nodesÚ.SIMDScheduling.generate_kernel_code_from_nodese  sZ  € ô Ñ2©EÓ2×2Ñ2Ü!$ UÑ0OÑ!P×!VÑ!VÑˆA‰�Ø ×7Ñ7¸ÀfÓMˆMØ×'Ñ'¨¸fÓEˆFØ×%Ñ%ØÜ+¨MÀ&ÓIð &ð ˆFð ×2Ñ2°=ÔIä—’Ð/Ð1AÕBÜ×$Ò$ VÕ,à!×0Ñ0Ó2�÷ -÷ CÐBð
 ,1°©8×+RÑ+RØó,Ñ(ˆH ô —’Ð0Ð2BÕCØ×0Ñ0ØØØØ&*Ø"/ð 1ð �÷ Dð ×#Ñ#¤C¬×(?Ñ(?Ó$@À)ÓLˆØˆ÷% -Õ,ú÷ CÕBú÷ DÕCús0   ÂEÂ/EÃ EÃ?E(Å
E	ÅEÅ
E%Å(
E6c                ó   • [         erX   rx  )rp   r\  r&  rn   s       r^   r�  ÚSIMDScheduling.define_kernel‡  r}  r`   rË   )rg  N)rb  zOptional[OrderedSet[str]]r…   ztuple[float, str]rX   )rš   z!Sequence[scheduler.SchedulerNode]r²  úOptional[CoalesceVarAnalysis])r¨   z<Union[scheduler.FusedSchedulerNode, scheduler.SchedulerNode])rl   r‚   rÃ  zGIterable[Union[ir.Buffer, ir.TensorBox, ir.TorchBindObject, ir.IRNode]]r…   rˆ   )F)rn   r   r&  úlist[NodeScheduleEntry]rb  rˆ   r…   r†   )rV  rJ   )rV  rJ   r…   zlist[SIMDKernel])r¨   r   r…   útuple[tuple[int, ...], ...])r¨   r   r…   rˆ   )r¨   r   r2  rÜ   r…   r7  )r8  rÞ   r…   úOptional[str])r[  zlist[BaseSchedulerNode]r\  rˆ   rR  rˆ   rS  rˆ   rb  rˆ   r…   z$list[tuple[Optional[str], Any, Any]])r…   rŽ  )r—  rá  r˜  rá  r…   úimmutable_dict[str, sympy.Expr])r(  rá  rƒ  rˆ   r…   r9  )r(  rÜ  rl   r‚   r(  r‚   r…   r9  )r…   z%list[immutable_dict[str, sympy.Expr]])
r&  r6  r«  r‚   r(  r‚   r²  rN   r…   ú=tuple[dict[str, sympy.Expr], Optional[dict[str, sympy.Expr]]])r&  r6  rl   r‚   r(  r‚   r(  rÜ  )r&  r6  rl   r‚   r(  r‚   rÀ  zlist[dict[str, sympy.Expr]])r²  r5  r…   rÜ  )r²  r5  r…   r:  r‡   )FN)r8  rÞ   )7rŠ   r‹   rŒ   r�   rŽ   rƒ   rë  r"  rð  rþ  Úcan_fuse_verticalr  r>  rC  rH  r]  rc  rB  r–  r·  rç  rÇ  rÊ  r±  rM  rN  r  r"  r-  r5  r@  rE  rf  rn  rè  rå   ræ   r”  r›  r  r�  rÁ  rþ  rð  r  r�   r�   r‘   r  rÏ  r$  r'  r1  r�  r“   rË   r`   r^   rê  rê    s  ‡ ñð
 (€K�Ó'òQò`6ðD !ÐØ"Ðò^ò@8ò
%ò2)ðj RVð"Ø5Nð"à	õ"ò
W)ðx <@ð
à0ð
ð 9õ
ð(=ØPô=ð2 ð$Øð$ð
ð$ð
 
ó$ó ð$ðT #(ð	@àð@ð /ð@ð  ð	@ð
 
õ@ô"Q)ðf
Ø1ð
à	ô
ò -ðT  õMð^Ø'ðà	$ôô,
ð
Ø'ð
Ø/2ð
à	$ô
ð.  Ø'+ñnð %ðnð 
õnò`Ið #(ði à0ði ð  $ði ð ð	i ð
 ði ð  ði ð 
.õi òV)ð( Ø×Ò˜Óó}ó ó ð}ð~ ð

Ø,ð

Ø@Tð

à	(ó

ó ð

ð ð
à$ð
ð ð
ð 
)ó	
ó ð
ð ð/à%ð/ð ð/ð $ð	/ð
 
)ó/ó ð/ð( ðyð
 
/óyó ðyðv ðY$à.ðY$ð $ðY$ð $ð	Y$ð
 /ðY$ð 
GóY$ó ðY$ðv ð
à.ð
ð ð
ð $ð	
ð
 &ó
ó ð
ð  ðà.ðð ðð $ð	ð
 4óó ðð ð
 Ÿ™Ÿ™Ø;?ð	ð
 9ð	ð 
ô	ó ð	ð ð
 Ÿ™Ÿ™Ø;?ðO$ð
 9ðO$ð 
GôO$ó ðO$òbôð MQð Ø<Iõ õD"r`   rê  T)Úfrozenc                  óH   • \ rS rSr% S\S'   S\S'   SrS\S'   \S	 5       rS
rg)r~  i‹  rÜ  r(  rÜ   ry  Nr8  ri   c                óz   • [         R                  R                  R                  U SS9n U S:¬  =(       a    U S-  S:H  $ )z@Somewhat arbitrary heuristic used to boost scores for some sizesi    rË  rp  r   )r=   r®   r¯   rÅ   )rØ   s    r^   r‚  ÚCandidateTiling.is_good_size‘  s:   € ô �G‰G×Ñ×.Ñ.¨q¸4Ð.Ð@ˆØ�B‰w×(˜A ™F a™KÐ(r`   rË   )	rŠ   r‹   rŒ   r�   r"  ri   rç  r‚  r“   rË   r`   r^   r~  r~  ‹  s)   ‡ à!Ó!ØƒJØ€Dˆ-Óàñ)ó ó)r`   r~  c                  ó.   ^ • \ rS rSrU 4S jrS rSrU =r$ )rþ  i˜  c                ó:   >• [         TU ]  5         Xl        X l        g rX   )rg   rh   r¶   r   )rp   r¶   r   rq   s      €r^   rh   ÚCantSplit.__init__™  s   ø€ Ü‰ÑÔØŒ	Ø"�r`   c                ó8   • U R                    SU R                   3$ )Nz not divisible by ©r¶   r   rt   s    r^   Ú__str__ÚCantSplit.__str__ž  s   € Ø—)‘)�Ð.¨t¯~©~Ð.>Ð?Ð?r`   rD  )rŠ   r‹   rŒ   r�   rh   rE  r“   r”   r•   s   @r^   rþ  rþ  ˜  s   ø† õ#÷
@ð @r`   rþ  )r   )r]   rÜ   r…   rÜ   )r  rå  r…   r   )ˆÚ
__future__r   r  rï  Údataclassesrå   rA  r  r  r  r  r   Útypingr   r   r   r   r	   r
   Útyping_extensionsr   r�   rY   Útorch._loggingÚtorch._inductorr   Útorch._inductor.irr   Útorch._inductor.tiling_utilsr   Ú%torch.fx.experimental.symbolic_shapesr   Útorch.fx.immutable_collectionsr   Útorch.utils._ordered_setr   Útorch.utils._sympy.functionsr   r   r   Útorch.utils._sympy.symbolr   r   r   r   Ú_dynamo.utilsr   Ú r   r   r   Úanalyze_preserves_zero_maskr   Ú	codecacher    r!   Údependenciesr"   r#   r$   Úcollections.abcr%   r'   Úoptimize_indexingr(   Ú runtime.coordinate_descent_tunerr)   Úruntime.hintsr*   Úruntime.runtime_utilsr+   r,   r-   r.   r/   r0   Úutilsr1   r2   r3   r4   r5   r6   r7   r8   r9   r:   Úvirtualizedr;   r<   r=   Úblock_analysisr?   Úcommonr@   rA   rB   rC   r?  rD   rE   Úsimd_kernel_featuresrF   rG   rH   rI   rJ   rK   rL   rM   rN   Ú	getLoggerrŠ   r­  Ú_loggingÚgetArtifactLoggerrñ  r¯  Ú
fusion_logÚdoprintræ  r–  r_   Ú	dataclassrb   r„   r±   r  r  r  r$  rƒ   rê  r~  Ú	Exceptionrþ  rË   r`   r^   Ú<module>rj     s  ðå "ã Û Û Û Û Û Û Û Û Ý ß K× KÝ %ã ã Û Ý #Ý 2Ý BÝ GÝ 9Ý /ß LÑ L÷ó õ &ß $Ñ $Ý Fß .ß 6Ñ 6ö Ý(åå AÝ <Ý ,ß LÑ Lß DÑ D÷÷ ÷ ÷ -Ñ ,Ý /ß PÓ Pß :÷õ ö ß<Ñ<å@ð ×Ò˜Ó!€Ø—‘×0Ñ0°¸<ÓH€Ø�~‰~×/Ñ/°¸*ÓE€Ø�^‰^×-Ñ-¨h¸ÓA€
ñ 	‹×Ñ€áÒ7Ó8€ö;ð
 ×Ñ÷3+ð 3+ó ð3+ôlH;˜/ô H;ôV;'˜?ô ;'ô|ñ Ð+°;ÈÑT€ð ×Ñ÷ð ó ðô
"ˆzô 
"ôG�˜Ñ(¨'°/Ñ*Bô GôTz"�^ô z"ðz; ×Ò˜dÑ#÷	)ð 	)ó $ð	)ô@�	õ @r`   