ó
    Eñið– ã                   ó	  • % S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	J
r
Jr  S SKJr  S SKJrJrJrJr  S SKrS SKrS SKrS SKJr  S SKJrJr  S SKJr  S SKJrJrJ r   S S	K!J"r"J#r#J$r$  S
SK%J&r&  SSK'J(r(J)r)J*r*J+r+J,r,  SSK-J.r.  SSK/J0r0  SSK1J2r2J3r3J4r4J5r5J6r6J7r7J8r8  SSK9J:r:J;r;J<r<J=r=J>r>J?r?J@r@JArAJBrBJCrCJDrDJErE  SSKFJGrGJHrHJIrIJJrJ  SSKKJLrLJMrMJNrNJOrOJPrPJQrQJRrRJSrSJTrTJUrUJVrVJWrW  SSKXJYrYJZrZJ[r[J\r\J]r]J^r^J_r_J`r`JaraJbrbJcrcJdrdJereJfrfJgrg  \RÐ                  S:H  ri\RÔ                  S 5       rk\RØ                  RÛ                  \nS5      ro\" / SQ5      rpSSSSSSS S!S"S"S#.
rq\" / S$Q5      rrS%S&S'S(S)S*S+S,S-S.S/.
rsS0S1S2.rt\Rê                  \Rì                  /rw\Rð                  \Rò                  \Rê                  \Rì                  \Rô                  \Rö                  \Rø                  \Rú                  \Rü                  \Rþ                  \GR                   /r�\‚\GR                     \„S3'   S4 r…S5 r†   SyS6\\GR                     4S7 jjrˆS8 r‰S9\SS:\GR                  S;\‹S<\GR                  S=\GR                  S>\M4S? jrŒS@\\‹\O4   SA\‹SB\‹SC\GR                  SD\\‹\�4   4
SE jrŽSF\SSG\‹SH\‹4SI jr�SF\S4SJ jr�\GR"                  S6\GR                  SK\GR                  4SL j5       r’\GR"                  S6\GR                  SK\GR                  SM\�4SN j5       r“\GR"                   SzS6\GR                  SK\GR                  SM\\�   4SO jj5       r”\GR*                   " SP SQ5      5       r– " SR SS\65      r— " ST SU5      r˜SV r™ " SW SX\V5      rš\šGR7                  SY5         " SZ S[\š5      rœ\œGR7                  S\5        \œGR;                  5          " S] S^\œ5      rž " S_ S`\T5      rŸ " Sa Sb\Ÿ5      r  " Sc Sd\ 5      r¡Se\0S>\¢\\GR                     \z4   4Sf jr£ " Sg Sh5      r¤ " Si Sj\Ÿ5      r¥ " Sk Sl\Ÿ5      r¦ " Sm Sn\5      r§ " So Sp\35      r¨ " Sq Sr5      r© " Ss St5      rª\GR*                   " Su Sv5      5       r«\GR*                   " Sw Sx5      5       r¬g){é    N)ÚCallableÚSequence)ÚEnum)ÚAnyÚcastÚOptionalÚUnion)Údependencies)Úis_float_dtypeÚis_integer_dtype)Ú
OrderedSet)ÚCeilDivÚFloorDivÚModularIndexing)Úfree_symbol_is_typeÚsymbol_is_typeÚSymTé   )Úcountersé   )ÚconfigÚcpp_builderÚcpu_vec_isaÚirÚmetrics)Ú'set_kernel_post_grad_provenance_tracing)ÚLoopBody)ÚBaseSchedulerNodeÚBaseSchedulingÚExternKernelSchedulerNodeÚForeachKernelSchedulerNodeÚFusedSchedulerNodeÚ	SchedulerÚSchedulerNode)Úcache_on_selfÚget_bounds_index_exprÚget_fused_kernel_nameÚhas_free_symbolsÚis_multi_outputs_templateÚis_welford_reductionÚparallel_num_threadsÚPlaceholderÚsympy_index_symbolÚsympy_index_symbol_with_prefixÚsympy_productÚ
sympy_subs)ÚNullKernelHandlerÚopsÚOpsValueÚVé   )ÚBackendFeatureÚBracesBufferÚCSEÚCSEVariableÚDataTypePropagationÚDeferredLineÚDTYPE_TO_COMPUTATION_DTYPEÚIndentedBufferÚKernelÚ
KernelArgsÚOpOverridesÚOptimizationContext)Ú_get_dtype_from_loopbodiesÚ_get_loop_bodyÚcexprÚcexpr_indexÚcodegen_randÚCppCSEVariableÚDTYPE_TO_CPPÚget_promote_dtypeÚ
INDEX_TYPEÚLocalBufferContextÚmay_unify_binary_op_mask_typeÚpromote_argsÚ(template_fusion_with_epilogues_supportedÚunify_mask_base_typeÚvalue_to_cppÚwin32c                  ó    • [         (       a  S$ S$ )Nz__declspec(dllexport)Ú ©Ú_IS_WINDOWS© ó    ÚX/home/mande/repo/quber/.venv/lib/python3.13/site-packages/torch/_inductor/codegen/cpp.pyÚget_export_declarationrY   Y   s   € ç&1¢kÐ"Ð9°rÐ9rW   Úschedule)Ú+Ú*Ú^ú||ÚminÚmaxr[   r\   r]   r_   r`   ÚargminÚargmaxr^   Úwelford)
ÚsumÚprodÚxor_sumr_   r`   ra   rb   ÚanyÚwelford_reduceÚwelford_combine)
r`   r_   rd   re   rf   rh   ri   ra   rb   rg   z
at::TensorÚlongÚdoubleÚboolzstd::stringzc10::ScalarTypezat::MemoryFormatz
at::Layoutz
at::Devicez
at::Scalar)
ÚTensorÚintÚfloatrl   ÚstrÚ
ScalarTypeÚMemoryFormatÚLayoutÚDeviceÚnumberzstd::vectorzstd::optional)ÚListr   ÚVECTORIZABLE_DTYPESc                 óÈ  • U[         ;   a  [        R                  nU S;   a  gU S:X  a  gU S;   a�  [        U   nU[        R                  :X  a  U S;   a  [        [        R
                     n[        U5      (       a  SU S3OS	U S
3n[        U5      (       a  S	U S3OS	U S3nU S;   a  UOUnU S;   a  U$ SU SU S3$ [        U 5      (       a  S[        U    S3$ [        U 5      e)N)rf   rd   rg   r   re   r5   )r`   rb   r_   ra   ©ra   rb   ú-std::numeric_limits<ú>::infinity()ústd::numeric_limits<ú>::min()ú>::max())r`   rb   )r`   r_   úIndexValue<z>{0, Ú}úWelford<ú>())	ÚDTYPE_LOWP_FPÚtorchÚfloat32rH   rl   ro   r   r*   ÚAssertionError)Úreduction_typeÚdtypeÚcdtypeÚmin_varÚmax_varÚinit_vars         rX   Úreduction_initr�   ¢   s$  € Ø”Óô —‘ˆØÐ2Ó2ØØ˜ÓØØÐ;Ó;Ü˜eÑ$ˆØ”E—J‘JÓ >Ð5IÓ#IÜ!¤%§+¡+Ñ.ˆFô ˜e×$Ñ$ð $ F 8¨=Ñ9à'¨ x¨xÐ8ð 	ô ˜e×$Ñ$ð # 6 (¨-Ñ8à'¨ x¨xÐ8ð 	ð
 -Ð0AÓA‘7Àwˆð  Ó/ð ð	
ð ˜v˜h f¨X¨J°bÐ9ð	
ô
 ˜N×+Ñ+Øœ, uÑ-Ð.¨cÐ2Ð2Ü
˜Ó
(Ð(rW   c                 óÀ   • [         [        U      n[        U 5      (       a  SU S3$ U S;   a1  U[        R                  :X  a  [         [        R
                     nSU S3$ U$ )Nr�   Ú>ry   r   )rH   r<   r*   r„   rl   ro   )r‡   rˆ   Úscalar_types      rX   Úreduction_acc_typer‘   Ä   sd   € ÜÔ9¸%Ñ@ÑA€KÜ˜N×+Ñ+Ø˜+˜ aÐ(Ð(ØÐ-Ó-Ø”E—J‘JÓÜ&¤u§{¡{Ñ3ˆKØ˜[˜M¨Ð+Ð+ØÐrW   Úindexc           	      óÀ  • U[         R                  :H  nU S:X  a%  U(       a	  SU SU S3$ U(       a  SOSnU SU SU 3$ U S:X  a  U S	U 3$ U S
:X  a  U SU 3$ U S:X  a  U SU 3$ U S;   a  U  SU SU S3$ U S:X  a  U(       a  SU SU SU S3$ SU SU S3$ U S:X  a8  [        U[        5      (       a  Uu  p‰n
O[	        X5      u  p‰n
SU SU SU	 SU
 S3	$ U S;   ax  [        US5      (       aK  UR                  [         R                  :X  a-  UR                  (       d  Ub  U  SU SU SU S3$ U  SU SU S3$ Ub  U  SU SU SU S3$ U  SU SU S3$ [        U 5      e)Nrd   úcascade_sum_combine(ú, &Ú)Ú|r[   Ú re   ú * rf   ú ^ rg   ú || )r_   r`   z_propagate_nan(ú, rh   úwelford_combine(ri   ú, {ú})ry   rˆ   z	_combine(z, static_cast<float>(ú), ú)))	r„   rl   Ú
isinstanceÚtupleÚreduction_projectÚhasattrrˆ   Úis_vecr†   )r‡   ÚvarÚ
next_valueÚ
helper_valr’   Ú	src_dtypeÚis_boolÚconjunctionÚmeanÚm2Úweights              rX   Úreduction_combiner°   Ï   s  € ð œ5Ÿ:™:Ñ%€GØ˜ÓÞØ)¨*¨°S¸¸ÀAÐFÐFæ!(™#¨cˆKØ�U˜!˜K˜=¨¨*¨Ð6Ð6Ø˜ÓØ��c˜*˜Ð&Ð&Ø˜Ó"Ø��c˜*˜Ð&Ð&Ø˜ÓØ��d˜:˜,Ð'Ð'Ø˜Ó'Ø Ð! °°°R¸
°|À1ÐEÐEØÐ)Ó)ÞØ% c U¨"¨Z¨L¸¸J¸<ÀqÐIÐIà% c U¨"¨Z¨L¸Ð:Ð:ØÐ*Ó*Ü�j¤%×(Ñ(Ø)ÑˆD‘fä0°ÓLÑˆD�fØ! #  d¨4¨&°°2°$°b¸¸ÀÐDÐDØÐ-Ó-ä�J ×(Ñ(Ø× Ñ ¤E§J¡JÓ.Ø×%×%àÑ Ø(Ð)¨°3°%Ð7LÈZÈLÐX[Ð\aÐ[bÐbcÐdÐdð &Ð& i°¨uÐ4IÈ*ÈÐUWÐXðð ÑØ$Ð% Y¨s¨e°2°j°\ÀÀEÀ7È!ÐLÐLà$Ð% Y¨s¨e°2°j°\ÀÐCÐCÜ
˜Ó
(Ð(rW   c                 óX   • [        U 5      (       a  U S3U S3U S34$ U S;   a  U S3$ U$ )Nz.meanz.m2z.weightry   z.index)r*   )r‡   Úaccs     rX   r¤   r¤     sF   € Ü˜N×+Ñ+Ø��eˆ}   S˜k¨c¨U°'¨?Ð:Ð:Ø	Ð/Ó	/Ø��fˆ~ÐØ€JrW   ÚcodeÚiter_varÚnew_iter_varÚ
loop_startÚloop_endÚreturnc                 óŒ  • [        5       n[        R                  " 5        nUR                  S[         SU S[        U5       S3U S[        U5       SU S3-   5        UR                  UR                  5       5        [        U R                  5       H”  u  px[        U[        [        45      (       d   eSn	[        U[        5      (       a  UR                  n	UR                  n[        R                   " S	U -   S	-   U U5      n
U	(       a  [        Xš5      n
UR                  U
5        M–     SSS5        U$ ! , (       d  f       U$ = f)
aÂ  
f(iter_var) is transformed to f(new_iter_var) under the inner loop
  \/
for (new_iter_var = loop_start; new_iter_var < loop_end; new_iter_var++) {
    f(new_iter_var)
}
Please be careful while using this function,
as the variable defined in f(iter_var) will be invalid outside the for loop.
For example:
auto tmp0 = in_ptr[x0]; ->
for (new_x0 = start; new_x0 < end; new_x0++){
    auto tmp0 = in_ptr[new_x0];
}
The tmp0 is invalid outside the loop.
zfor (r˜   ú = Ú;ú < ú; ú++)Nú\b)r7   Ú
contextlibÚ	ExitStackÚ	writelinerJ   rE   Úenter_contextÚindentÚ	enumerateÚ_linesr¢   rp   r;   ÚnameÚlineÚreÚsub)r³   r´   rµ   r¶   r·   Útransformed_codeÚstackÚ_rÈ   Údeferred_nameÚnew_lines              rX   Úmove_code_under_inner_looprÐ     s<  € ô, $“~ÐÜ	×	Ò	Ô	 5Ø×"Ñ"Ø”J�<˜q  ¨c´+¸jÓ2IÐ1JÈ!ÐLØ�˜c¤+¨hÓ"7Ð!8¸¸<¸.ÈÐLñMô	
ð 	×ÑÐ,×3Ñ3Ó5Ô6Ü  §¡Ö-‰GˆAÜØäÜ ð÷ñ ð ð ð !ˆMÜ˜$¤×-Ñ-Ø $§	¡	�Ø—y‘y�Ü—v’v˜e¨¨
Ñ3°eÑ;À¸~ÐPTÓUˆHÞÜ'¨Ó@�Ø×&Ñ& xÖ0ñ .÷ 
 ð, Ð÷- 
 Ô	ð, Ðús    D
D4Ä4
EÚacc_varÚacc_typer‡   rˆ   Úlenc                 óì   • [        5       n[        R                  " 5       (       a  SU  SU SU S3O
U SU  SU S3nUR                  U 5        UR	                  SU S	3S
SU  SU" X#5       S3S/5        U$ )a€  
MSVC don't support dynamic array(VLA). So we use std::unique_ptr here.
Ref: https://stackoverflow.com/questions/56555406/creating-dynamic-sized-array-using-msvc-c-compiler
MSVC is the only one compiler without VLA. support. Since MSVC can't get good performance here.
We just use unique_ptr make it works on MSVC.
For other compilers, we continue to use VLA to get best performance.
úauto z_arr = std::make_unique<z[]>(ú);r˜   ú_arr[ú];úfor (int i = 0; i < ú; i++)Ú{ú    z
_arr[i] = r»   r€   )r=   r   Ú
is_msvc_clrÂ   Ú
writelines)rÑ   rÒ   r‡   rˆ   rÓ   Úinit_fnÚcode_bufferÚacc_decls           rX   Úreduction_prefix_arrayrâ   <  sª   € ô !Ó"€Kô ×!Ò!×#Ñ#ð �ˆyÐ0°°
¸$¸s¸eÀ2ÑFàˆz˜˜7˜) 5¨¨¨RÐ0ð ð
 ×Ñ˜X˜JÔ(Ø×Ñà" 3 % vÐ.ØØ�7�)˜:¡g¨nÓ&DÐ%EÀQÐGØð		
ôð ÐrW   ÚbufferrÇ   Únew_namec                 ó`  • [        U R                  5       H•  u  p4[        U[        [        45      (       d   e[        U[        5      (       a1  [
        R                  " SU -   S-   U UR                  5      Ul        Mh  [
        R                  " SU -   S-   U U5      U R                  U'   M—     g )Nr¿   )rÅ   rÆ   r¢   rp   r;   rÉ   rÊ   rÈ   )rã   rÇ   rä   ÚirÈ   s        rX   Úreplace_acc_namerç   ]  s›   € Ü˜VŸ]™]Ö+‰ˆÜØäÜð÷
ñ 
ð 	
ð 
ô �dœL×)Ñ)ÜŸš˜u¨$¨Ñ0°5Ñ8¸X¸JÈÏÉÓSˆDŽIä!Ÿvšv e°°Ñ&7¸%Ñ&?ÀHÀ:ÐPTÓUˆF�M‰M˜!Óò ,rW   c           
      ó¼  • Sn[        U R                  5       HÁ  u  p#[        U[        [        45      (       d   e[        U[        5      (       a  UR
                  OUn[        R                  " X5      nU(       d  Md  UR                  5       u  pg[        R                  " X SU SU S3U5      n[        U[        5      (       a  Xƒl        M³  X€R                  U'   MÃ     g)zL
Replaces `acc = cascade_sum_combine(value, ...)` with `acc = acc + value;`
z/(.*?)\s*=\s*cascade_sum_combine\(([^,]+),.*?\);rº   ú + r»   N)
rÅ   rÆ   r¢   rp   r;   rÈ   rÉ   ÚsearchÚgroupsrÊ   )	rã   Úpatternræ   rÈ   ÚcontentÚmatchr²   ÚvalueÚnew_contents	            rX   Úreplace_cascade_sum_with_addrñ   l  s¾   € ð
 A€GÜ˜VŸ]™]Ö+‰ˆÜØäÜð÷
ñ 
ð 	
ð 
ô  *¨$´×=Ñ=�$—)’)À4ˆÜ—	’	˜'Ó+ˆßˆ5ØŸ™›‰JˆCÜŸ&š& ¨E°°S°E¸¸U¸GÀ1Ð*EÀwÓOˆKÜ˜$¤×-Ñ-Ø'–	à#.—‘˜aÓ ò! ,rW   r§   c                 ó´   • U R                  U5      (       d  [        R                  R                  $ XS-   0n[	        X5      n[        R
                  " X0-
  5      $ ©Nr5   )ÚhasÚsympyÚSÚZeror0   Úsimplify)r’   r§   ÚreplacementÚ	new_indexs       rX   Ú	stride_atrû   …  sF   € à�9‰9�S�>‰>ô �w‰w�|‰|ÐØ˜a™�.€KÜ˜5Ó.€IÜ�>Š>˜)Ñ+Ó,Ð,rW   Ú
vec_lengthc                 ó¶  ^^^^	• SmSm	UUU4S jnU	UU4S jnU n[         R                  " SSS9nU R                  [        5      (       a  U R	                  [        TU5      U5      n [         R                  " SSS9nU R                  [
        5      (       a  U R	                  [        TXg5      U5      n [         R                  " U 5      n X:w  a  [        U TT5      $ U $ )ai  
Simplifies the index expression within the range of a vectorized loop.
Given a vectorized loop variable `var` in the range of a loop with `vec_length`,
this function transforms the `index` into an equivalent form. It handles
simplifications for cases where `var` can be expressed as `vec_length * a + b`,
where `b` ranges from 0 to `vec_length - 1`. The function reduces occurrences
of `FloorDiv` and `ModularIndexing` in the `index` with best-effort optimizations.

NOTE:
The simplified index expression is intended for analysis purposes only, not
for code generation. It replaces `FloorDiv` and `ModularIndexing` with free variables
which are not dependent on the loop variable `var` in the vectorized range. Check
https://github.com/pytorch/pytorch/pull/117221#discussion_r1449746217 for more details.

Examples:
1. If `var` is `x3` and `vec_length` is 16, and `x3 = 16*a + b`, then
   `FloorDiv(x3, div)` or `ModularIndexing(x3, div, mod)` becomes a free variable
   when `div` is divisible by 16.
2. `ModularIndexing(x3, 1, mod)` can be simplified to `x3 + c` where `c` is a free
   variable when `mod` is divisible by 16.
r   c                 ó–   >• [        TU 5      n[        R                  " U T5      T:X  a   [        R                  " T ST 35      nTS-  mU$ )NÚ_div_cr5   )r   rõ   ÚgcdÚSymbol)ÚdivisorÚresultÚdiv_freevar_idr§   rü   s     €€€rX   Úvisit_indexing_divÚ7simplify_index_in_vec_range.<locals>.visit_indexing_div¬  sK   ø€ ä˜#˜wÓ'ˆÜ�9Š9�W˜jÓ)¨ZÓ7Ü—\’\ S E¨°Ð/?Ð"@ÓAˆFØ˜aÑˆNØˆrW   c                 ó"  >• [        TX5      n[        R                  " U T5      T:X  a"  [        R                  " T ST 35      nTS-  mU$ U S:X  a>  [        R                  " UT5      T:X  a#  T[        R                  " T ST 35      -   nTS-  mU$ )NÚ_mod_cr5   )r   rõ   r   r  )r  Úmodulusr  Úmod_freevar_idr§   rü   s      €€€rX   Úvisit_modular_indexingÚ;simplify_index_in_vec_range.<locals>.visit_modular_indexing´  s–   ø€ ä   gÓ7ˆÜ�9Š9�W˜jÓ)¨ZÓ7Ü—\’\ S E¨°Ð/?Ð"@ÓAˆFØ˜aÑˆNð ˆð ˜‹\œeŸiši¨°Ó<À
ÓJØœ5Ÿ<š<¨3¨%¨v°nÐ5EÐ(FÓGÑGˆFØ˜aÑˆNØˆrW   r  T)Úintegerr	  )rõ   ÚWildrô   r   Úreplacer   rø   Úsimplify_index_in_vec_range)
r’   r§   rü   r  r  Úoriginal_indexÚdivÚmodr  r
  s
    ``     @@rX   r  r  ‘  s¶   û€ ð0 €NØ€N÷÷	ð €Nä
�*Š*�Y¨Ñ
-€CØ‡y�y”×ÑØ—‘œh s¨CÓ0Ð2DÓEˆä
�*Š*�Y¨Ñ
-€CØ‡y�y”×!Ñ!Ø—‘œo¨c°3Ó<Ð>TÓUˆä�NŠN˜5Ó!€EØÓÜ*¨5°#°zÓBÐBà€LrW   c                 ó>   • U(       a  [        XU5      n [        X5      $ ©N)r  rû   )r’   r§   rü   s      rX   Ústride_at_vec_ranger  Ð  s   € ö Ü+¨E¸
ÓCˆÜ�UÓ Ð rW   c                   ó.   • \ rS rSr% Sr\\S'   \\S'   Srg)ÚParallelDepthiÙ  zo
A class representing parallel depth.
Includes the starting depth of parallelism and the depth of parallelism.
Úparallel_depthÚstart_depthrV   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rn   Ú__annotations__Ú__static_attributes__rV   rW   rX   r  r  Ù  s   ‡ ñð
 ÓØÖrW   r  c                   ór   ^ • \ rS rSr\S\S\4S j5       rSSS\\\	\
4      4U 4S jjrS	 rS
 rS rSrU =r$ )ÚOuterLoopFusedSchedulerNodeiä  Únode1Únode2c                 ó²  • UR                   UR                   L d   e[        S X4 5       5      (       d   e[        S X4 5       5      (       av  U " UR                   [        U5      [        L a  [        UR                  5       5      OU/[        U5      [        L a   [        UR                  5       5      -   U5      $ U/-   U5      $ U " UR                   X/U5      $ )Nc              3   ó\   #   • U  H"  n[        U5      [        [        [        4;   v •  M$     g 7fr  )Útyper#  r$   r"   ©Ú.0Únodes     rX   Ú	<genexpr>Ú3OuterLoopFusedSchedulerNode.fuse.<locals>.<genexpr>ê  s2   é € ð 
ò '�ô �‹Jä+ÜÜ"ðöò 'ùó   ‚*,c              3   óD   #   • U  H  n[        U5      [        L v •  M     g 7fr  ©r(  r#  r)  s     rX   r,  r-  ó  ó   é € ÐTÂ^¸TŒt�D‹zÔ8Õ8Â^ùó   ‚ )Ú	schedulerÚallrg   r(  r#  ÚlistÚget_outer_nodes)Úclsr$  r%  Úouter_loop_fusion_depths       rX   ÚfuseÚ OuterLoopFusedSchedulerNode.fuseå  sù   € ð �‰ %§/¡/Ò1Ð1Ð1Üñ 
ð ™ó
÷ 
ñ 
ð 	
ð 
ô ÑTÀeÁ^ÓT×TÑTÙØ—‘ô ˜E“{Ô&AÒAô ˜×.Ñ.Ó0Ô1ð ðô ˜E“{Ô&AÒAô ˜×.Ñ.Ó0Ó1ñð (ó#ð ð ðñð (ó#ð ñ( �u—‘¨¨Ð8OÓPÐPrW   r3  r#   Úouter_fused_nodesc                 óò   >• UU l         X0l        / nU R                    HH  n[        U[        [        45      (       d   eUR                  [        UR                  5       5      5        MJ     [        TU ]%  X5        g r  )
r;  r8  r¢   r$   r"   Úextendr5  Ú	get_nodesÚsuperÚ__init__)Úselfr3  r;  r8  Úflatten_snodesÚ_nodeÚ	__class__s         €rX   r@  Ú$OuterLoopFusedSchedulerNode.__init__
  so   ø€ ð ð 	Ôð (?Ô$ØˆØ×+Ô+ˆEÜ˜e¤mÔ5GÐ%H×IÑIÐIÐIØ×!Ñ!¤$ u§¡Ó'8Ó"9Ö:ñ ,ô 	‰Ñ˜Õ3rW   c                 ó   • U R                   $ r  )r;  ©rA  s    rX   r6  Ú+OuterLoopFusedSchedulerNode.get_outer_nodes  s   € Ø×%Ñ%Ð%rW   c           
      ój  ^• S[         S[         S[        S[        S[        4
U4S jjm[        [	        U5      S-
  5       H4  nX   R
                  nXS-      R
                  nT" UUUS5      (       a  M4    g	   U H·  n[        R                  " [        R                  UR                  S U 5      n[	        UR                  5      U:”  d  MP  [        U[        R                  5      (       d  Mq  [        UR                  U   [        R                  5      (       d  MŸ  US
-  UR                  U   :  d  M·    g	   g)NÚleft_loop_nestÚright_loop_nestÚloop_fusion_depthÚcurrent_checking_depthr¸   c                 ó�  >^^• U R                   (       d   eUR                   (       d   eU R                   U   mUR                   U   m/ SQn[        UU4S jU 5       5      (       d  gUS:¼  d   eUS-
  =nS:”  aL  US-   nU[        U R                   5      :  d   eU[        UR                   5      :  d   eT" U UUU5      (       d  gg)N)r§   ÚsizeÚoffsetÚstepsc              3   óV   >#   • U  H  n[        TU5      [        TU5      :H  v •  M      g 7fr  )Úgetattr)r*  Úattr_compareÚleft_loop_levelÚright_loop_levels     €€rX   r,  ÚaOuterLoopFusedSchedulerNode.check_outer_fusion_loop_level_attr.<locals>._inner.<locals>.<genexpr>8  s2   øé € ð ò )F˜ô ˜O¨\Ó:ÜÐ/°Ó>ö?â(Eùs   ƒ&)Fr5   r   T)Úloopsr4  rÓ   )rJ  rK  rL  rM  Úouter_loops_attr_compare_listrU  rV  Ú_inners        @@€rX   rZ  ÚNOuterLoopFusedSchedulerNode.check_outer_fusion_loop_level_attr.<locals>._inner&  sñ   ú€ ð "×'×'Ð'Ð'Ø"×(×(Ð(Ð(Ø,×2Ñ2Ð3IÑJˆOØ.×4Ñ4Ð5KÑLÐò-Ð)ô õ ñ )Fó÷ ñ ð à$¨Ó)Ð)Ð)Ø%6¸Ñ%:Ð:Ð!¸aÓ?à)?À!Ñ)CÐ&Ø-´°N×4HÑ4HÓ0IÓIÐIÐIØ-´°O×4IÑ4IÓ0JÓJÐJÐJÙØ"Ø#Ø%Ø*÷	ñ ð !àrW   r5   r   Fé,  T)ÚLoopNestrn   rl   ÚrangerÓ   Ú	loop_nestÚ	functoolsÚreduceÚoperatorÚmulÚrangesr¢   rõ   ÚInteger)	rA  Úcpp_kernel_proxy_listr8  ÚidxrJ  rK  Úcpp_kernel_proxyÚouter_rangesrZ  s	           @rX   Ú"check_outer_fusion_loop_level_attrÚ>OuterLoopFusedSchedulerNode.check_outer_fusion_loop_level_attr  s;  ø€ ð(	Ü$ð(	ä%ð(	ô  #ð(	ô %(ð	(	ô
 ÷(	ôT œÐ2Ó3°aÑ7Ö8ˆCØ2Ñ7×AÑAˆNØ3¸!±GÑ<×FÑFˆOÙØØØ'Ø÷	ó ñ ñ 9ó !6ÐÜ$×+Ò+Ü—‘Ø ×'Ñ'Ð(@Ð)@ÐAóˆLô Ð$×+Ñ+Ó,Ð/FÕFÜ˜|¬U¯]©]×;Ó;ÜØ$×+Ñ+Ð,CÑDÜ—M‘M÷ó ð ! 3Ñ&Ø"×)Ñ)Ð*AÑBõCñ ñ) !6ð, rW   c                 óH  • US   R                   n[        U5      nU Vs/ s H(  nUR                  R                  U R                  5      PM*     snUl        US   nX5R                  l        UR                  R                  S U R                   UR                  l        U$ s  snf ©Nr   )Úkernel_groupÚOuterLoopFusedKernelr_  Úfrom_loop_levelr8  ÚinnerÚkernelrX  )rA  rf  rn  Úouter_loop_fused_kernelÚproxyÚouter_fused_proxys         rX   Úmerge_outer_fusion_kernelsÚ6OuterLoopFusedSchedulerNode.merge_outer_fusion_kernelss  s¨   € ð -¨QÑ/×<Ñ<ˆÜ"6°|Ó"DÐñ /ó)
â.�ð �O‰O×+Ñ+¨D×,HÑ,HÖIÙ.ñ)
ÐÔ%ð 2°!Ñ4ÐØ-D×#Ñ#Ô*Ø,=×,GÑ,G×,MÑ,MØ*ˆd×*Ñ*ð-
Ð×#Ñ#Ô)ð !Ð ùò)
s   Ÿ/B)r;  r8  )r  r  r  r  Úclassmethodr   r9  r5  r	   r"   r$   r@  r6  rj  rv  r!  Ú__classcell__©rD  s   @rX   r#  r#  ä  sg   ø† Øð"QØ%ð"QØ.?ó"Qó ð"QðH4àð4ð   Ð&8¸-Ð&GÑ HÑI÷4ò &òT÷l!ð !rW   r#  c                   ó>   • \ rS rSrS
S\4S jjrS rS rS rS r	Sr
g	)ÚRecordOptimizationContexti…  Ú	func_namec                 ó,   • Xl         S U l        S U l        g r  )r}  Úcurrent_nodeÚopt_ctx)rA  r}  s     rX   r@  Ú"RecordOptimizationContext.__init__†  s   € Ø"ŒØ59ˆÔØ6:ˆ�rW   c                 óî  • [         R                  (       d   e[         R                  R                  (       d   e[         R                  R                  U l        U R                  c   e[        R                  U R                  R
                  ;   a-  U R                  R
                  [        R                     U l        O[        5       U l        U R                  c   eU R                  U R                  l        U $ r  )	r4   Úinterpreterr  rA   ÚkeyÚmetar€  r}  Úops_namerG  s    rX   Ú	__enter__Ú#RecordOptimizationContext.__enter__‹  s­   € Ü�}�}Ðˆ}Ü�}‰}×)×)Ð)Ð)äŸM™M×6Ñ6ˆÔØ× Ñ Ñ,Ð,Ð,Ü×"Ñ" d×&7Ñ&7×&<Ñ&<Ó<Ø×,Ñ,×1Ñ1Ô2E×2IÑ2IÑJˆD�Lä.Ó0ˆDŒLØ�|‰|Ñ'Ð'Ð'Ø $§¡ˆ�‰ÔØˆrW   c                 ó²   • U R                   (       d   eU R                  (       d   eU R                  U R                   R                  [        R                  '   g r  )r  r€  r…  rA   r„  ©rA  Úexc_typeÚexc_valÚexc_tbs       rX   Ú__exit__Ú"RecordOptimizationContext.__exit__™  s>   € Ø× × Ð Ð Ø�|�|Ðˆ|Ø:>¿,¹,ˆ×Ñ×ÑÔ2×6Ñ6Ò7rW   c                 ó   • U R                   $ r  )r€  rG  s    rX   Úget_opt_ctxÚ%RecordOptimizationContext.get_opt_ctxž  s   € Ø�|‰|ÐrW   c                 ó@   • U R                   (       d   eU R                   $ r  )r  rG  s    rX   Úget_fx_nodeÚ%RecordOptimizationContext.get_fx_node¡  s   € Ø× × Ð Ð Ø× Ñ Ð rW   )r  r}  r€  N)rS   )r  r  r  r  rp   r@  r‡  rŽ  r‘  r”  r!  rV   rW   rX   r|  r|  …  s#   † ñ; #õ ;ò
òGò
õ!rW   r|  c                  ó~   • [        S U  5       5      (       a   S5       e[        U 5      =nb	  [        U   $ SU S    S3$ )Nc              3   óh   #   • U  H(  n[        U[        5      =(       a    UR                  v •  M*     g 7fr  )r¢   rG   r¦   )r*  Úargs     rX   r,  Ú$decltype_promoted.<locals>.<genexpr>§  s#   é € ÐRÊTÀc”:˜c¤>Ó2×A°s·z±zÔAÊTùs   ‚02z*Promotion of vector types is not supportedú	decltype(r   r–   )rg   rI   rH   )ÚargsÚdts     rX   Údecltype_promotedr�  ¦  sS   € ÜÑRÉTÓR×RÑRð Ø4óÐRô   Ó%Ð%ˆÑ2Ü˜BÑÐà˜4 ™7˜) 1Ð%Ð%rW   c            
       ó`  • \ rS rSrSr\S 5       r\S 5       r\S 5       r\SUS j5       r	\S 5       r
\S	 5       r\S
 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r \S 5       r!\S  5       r"\S! 5       r#\S" 5       r$\S# 5       r%\S$ 5       r&\S% 5       r'\S& 5       r(\S' 5       r)\S( 5       r*\S) 5       r+\S* 5       r,\S+ 5       r-\S, 5       r.\S- 5       r/\S. 5       r0\S/ 5       r1\S0 5       r2\S1 5       r3\S2 5       r4\S3 5       r5\S4 5       r6\S5 5       r7\S6 5       r8\S7 5       r9\S8 5       r:\S9 5       r;\S: 5       r<\S; 5       r=\S< 5       r>\S= 5       r?\S> 5       r@\S? 5       rA\S@ 5       rB\SA 5       rC\SB 5       rD\SC 5       rE\SD 5       rF\SE 5       rG\SF 5       rH\SG\IR”                  SH\IR”                  4SI j5       rK\SG\IR”                  SH\IR”                  4SJ j5       rL\SG\IR”                  SH\IR”                  4SK j5       rM\SL 5       rN\SM 5       rOSN\PSO\PSP\QSQ\R\P\S4   SRS4
SS jrTSTrUg)VÚCppOverridesi±  zMap element-wise ops to C++c                 ó*   • [        X5       SU  SU S3$ )NÚ(ré   r–   ©r�  ©ÚaÚbs     rX   ÚaddÚCppOverrides.add´  ó!   € ä# AÓ)Ð*¨!¨A¨3¨c°!°°AÐ6Ð6rW   c                 ó*   • [        X5       SU  SU S3$ )Nr¡  ú - r–   r¢  r£  s     rX   rÊ   ÚCppOverrides.sub¸  r¨  rW   c                 ó*   • [        X5       SU  SU S3$ )Nr¡  r™   r–   r¢  r£  s     rX   rc  ÚCppOverrides.mul¼  r¨  rW   Nc                 óÀ  • [        U [        5      (       d   eUc  U R                  n[        R                  R                  XU5      n[        R                  R                  R                  [        R                  R                  U5      nUR                  SX4SU05        U[        ;   a5  U[        R                  :X  a!   [        R                  R                  XXQ5        U$ )NÚto_dtyperª   )r¢   rG   rˆ   r4   rr  Úget_to_dtype_exprÚcseÚgenerateÚcomputeÚupdate_on_argsrƒ   r„   ro   Úcache_dtype_convert©Úxrˆ   rª   Úuse_compute_typesÚexprÚcsevars         rX   r¯  ÚCppOverrides.to_dtypeÀ  sª   € ä˜!œ^×,Ñ,Ð,Ð,ØÑØŸ™ˆIÜ�x‰x×)Ñ)¨!°IÓ>ˆÜ—‘—‘×&Ñ&¤q§x¡x×'7Ñ'7¸Ó>ˆØ×Ñ˜j¨1¨*°{ÀIÐ6NÔOØ”MÓ! i´5·;±;Ó&>ðô> �H‰H×(Ñ(¨°vÔEØˆrW   c                 óX   • U[         ;   d   U S[         S35       eS[         U    SU  S3$ )Nz missing from z.DTYPE_TO_CPPzc10::bit_cast<ú>(r–   )rH   r  )r·  rˆ   rª   s      rX   Úto_dtype_bitcastÚCppOverrides.to_dtype_bitcastë  s=   € àœÓ$ÐU¨¨¨~¼h¸ZÀ}Ð&UÓUÐ$Ø¤¨UÑ 3Ð4°B°q°c¸Ð;Ð;rW   c                 ó   • SU  S3$ )Nz	std::abs(r–   rV   ©r·  s    rX   ÚabsÚCppOverrides.absð  ó   € à˜1˜#˜QÐÐrW   c                 ó   • SU  S3$ )Nz	std::sin(r–   rV   rÁ  s    rX   ÚsinÚCppOverrides.sinô  rÄ  rW   c                 ó   • SU  S3$ )Nz	std::cos(r–   rV   rÁ  s    rX   ÚcosÚCppOverrides.cosø  rÄ  rW   c                 ó   • SU  SU  S3$ )Nrš  z)(-r–   rV   rÁ  s    rX   ÚnegÚCppOverrides.negü  ó   € à˜1˜#˜S   1Ð%Ð%rW   c                 ó   • SU  S3$ )Nz	std::exp(r–   rV   rÁ  s    rX   ÚexpÚCppOverrides.exp   s   € ð ˜1˜#˜QÐÐrW   c                 ó   • SU  S3$ )Nz
std::exp2(r–   rV   rÁ  s    rX   Úexp2ÚCppOverrides.exp2  ó   € à˜A˜3˜aÐ Ð rW   c                 ó   • SU  S3$ )Nzstd::expm1(r–   rV   rÁ  s    rX   Úexpm1ÚCppOverrides.expm1	  ó   € à˜Q˜C˜qÐ!Ð!rW   c                 ó   • SU  S3$ )Nz	std::erf(r–   rV   rÁ  s    rX   ÚerfÚCppOverrides.erf  rÄ  rW   c                 ó   • SU  S3$ )Nz
std::erfc(r–   rV   rÁ  s    rX   ÚerfcÚCppOverrides.erfc  rÕ  rW   c                 ó   • SU  S3$ )Nzcalc_erfinv(r–   rV   rÁ  s    rX   ÚerfinvÚCppOverrides.erfinv  ó   € à˜a˜S Ð"Ð"rW   c                 ó   • SU  S3$ )Nz
std::sqrt(r–   rV   rÁ  s    rX   ÚsqrtÚCppOverrides.sqrt  rÕ  rW   c                 ó   • SU  S3$ )Nz1 / std::sqrt(r–   rV   rÁ  s    rX   ÚrsqrtÚCppOverrides.rsqrt  s   € à ˜s !Ð$Ð$rW   c                 ó‚   • [         R                  R                  nUS:X  a  U  SU  S3$ Uc  SU  S3$ [        SU< 35      e)NÚaccuracyú + decltype(ú)(1)zstd::log1p(r–   ú8unrecognized config cpp.inject_log1p_bug_TESTING_ONLY = ©r   ÚcppÚinject_log1p_bug_TESTING_ONLYr†   ©r·  Úbugs     rX   Úlog1pÚCppOverrides.log1p!  sW   € ä�j‰j×6Ñ6ˆØ�*ÓØ�S˜ Q C tÐ,Ð,Ø‰[Ø    1Ð%Ð%ä ØJÈ3É'ÐRóð rW   c                 ó   • SU  S3$ )Nz	std::tan(r–   rV   rÁ  s    rX   ÚtanÚCppOverrides.tan-  rÄ  rW   c                 ó   • SU  S3$ )Nz
std::tanh(r–   rV   rÁ  s    rX   ÚtanhÚCppOverrides.tanh1  rÕ  rW   c                 ó0   • [         (       a  SU  S3$ SU  S3$ )z�
On windows std::signbit only support float type.
Ref: https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/signbit?view=msvc-170
z std::signbit(static_cast<float>(r¡   zstd::signbit(r–   rT   rÁ  s    rX   ÚsignbitÚCppOverrides.signbit5  s/   € ÷ Š{ð /¨q¨c°Ð4ð	
ð !   1Ð%ð	
rW   c                 ó   • SU  SU S3$ )Nz	std::pow(rœ   r–   rV   r£  s     rX   ÚpowÚCppOverrides.powA  s   € à˜1˜#˜R ˜s !Ð$Ð$rW   c                 ó   • SU  S3$ )Nz	std::log(r–   rV   rÁ  s    rX   ÚlogÚCppOverrides.logE  rÄ  rW   c                 ó   • SU  S3$ )Nzstd::nearbyint(r–   rV   rÁ  s    rX   ÚroundÚCppOverrides.roundI  s   € à    1Ð%Ð%rW   c                 ó   • SU  S3$ )Nzstd::floor(r–   rV   rÁ  s    rX   ÚfloorÚCppOverrides.floorM  rÙ  rW   c                 óH   • U  SU 3nU  SU 3nSU  SU SU SU SU SU S	3$ )
Nú / ú % ú((z
 < 0) != (z	 < 0) ? (z != 0 ? z - 1 : z) : r–   rV   )r¤  r¥  ÚquotÚrems       rX   ÚfloordivÚCppOverrides.floordivQ  sR   € ð ��C˜�sˆ|ˆØ��3�q�cˆlˆØ�A�3�j   9¨S¨E°¸$¸¸wÀtÀfÈDÐQUÐPVÐVWÐXÐXrW   c                 ó   • SU  S3$ )Nz
std::ceil(r–   rV   rÁ  s    rX   ÚceilÚCppOverrides.ceilX  rÕ  rW   c                 ó   • SU  S3$ )Nzstd::trunc(r–   rV   rÁ  s    rX   ÚtruncÚCppOverrides.trunc\  rÙ  rW   c                 ó   • U  SU 3$ ©Nr  rV   r£  s     rX   ÚtruncdivÚCppOverrides.truncdiv`  s   € ð ��C˜�sˆ|ÐrW   c                 ó   • SU  SU S3$ )Nz
std::fmod(rœ   r–   rV   r£  s     rX   ÚfmodÚCppOverrides.fmode  s   € à˜A˜3˜b   1Ð%Ð%rW   c                 ó   • SU  S3$ )Nzstd::isinf(r–   rV   rÁ  s    rX   ÚisinfÚCppOverrides.isinfi  rÙ  rW   c                 ó   • SU  S3$ )Nzstd::isnan(r–   rV   rÁ  s    rX   ÚisnanÚCppOverrides.isnanm  rÙ  rW   c                 ó   • SU  S3$ )Nzstd::lgamma(r–   rV   rÁ  s    rX   ÚlgammaÚCppOverrides.lgammaq  rã  rW   c                 ó   • SU  S3$ )Nz
std::acos(r–   rV   rÁ  s    rX   ÚacosÚCppOverrides.acosu  rÕ  rW   c                 ó   • SU  S3$ )Nzstd::acosh(r–   rV   rÁ  s    rX   ÚacoshÚCppOverrides.acoshy  rÙ  rW   c                 ó   • SU  S3$ )Nz
std::cosh(r–   rV   rÁ  s    rX   ÚcoshÚCppOverrides.cosh}  rÕ  rW   c                 ó   • SU  S3$ )Nz
std::sinh(r–   rV   rÁ  s    rX   ÚsinhÚCppOverrides.sinh�  rÕ  rW   c                 ó   • SU  S3$ )Nz
std::asin(r–   rV   rÁ  s    rX   ÚasinÚCppOverrides.asin…  rÕ  rW   c                 ó   • SU  S3$ )Nzstd::asinh(r–   rV   rÁ  s    rX   ÚasinhÚCppOverrides.asinh‰  rÙ  rW   c                 ó   • SU  SU S3$ )Nzstd::atan2(rœ   r–   rV   ©r·  Úys     rX   Úatan2ÚCppOverrides.atan2�  ó   € à˜Q˜C˜r !  AÐ&Ð&rW   c                 ó   • SU  S3$ )Nz
std::atan(r–   rV   rÁ  s    rX   ÚatanÚCppOverrides.atan‘  rÕ  rW   c                 ó   • SU  S3$ )Nzstd::atanh(r–   rV   rÁ  s    rX   ÚatanhÚCppOverrides.atanh•  rÙ  rW   c                 ó   • SU  SU S3$ )Nzstd::copysign(rœ   r–   rV   r<  s     rX   ÚcopysignÚCppOverrides.copysign™  s   € à ˜s " Q C qÐ)Ð)rW   c           	      ó°  • SU  S3SU  S34n[        S U 5       5      (       a  [        S U 5       5      $ [        5       n[        R                  R
                  R                  [        R                  U R                  S9n[        R                  R
                  R                  U R                  U R                  S9nUR                  SU S35        UR                  S	U S
U  SU S35        [        R                  R                  R                  U5        XC4n[        X5       H.  u  pg[        R                  R
                  R                  Xg5        M0     XC4$ )Núfrexp(ú)[0]ú)[1]c              3   óx   #   • U  H0  n[         R                  R                  R                  U5      S Lv •  M2     g 7fr  ©r4   rr  r±  Útry_get©r*  Ú	cache_keys     rX   r,  Ú%CppOverrides.frexp.<locals>.<genexpr>   ó(   é € ÐWÊJ¸yŒq�x‰x�|‰|×#Ñ# IÓ.°dÕ:ÊJùó   ‚8:c              3   ót   #   • U  H.  n[         R                  R                  R                  U5      v •  M0     g 7fr  rO  rQ  s     rX   r,  rS  ¡  ó&   é € ÐUÊ*¸YœŸ™Ÿ™×-Ñ-¨i×8Ð8Ê*ùó   ‚68)rˆ   Úshapezint32_t r»   rÕ   z = std::frexp(r•   rÖ   )r4  r£   r7   r4   rr  r±  Únewvarr„   Úint32rY  rˆ   rÂ   r³  ÚspliceÚzipÚput)r·  Ú
cache_keysr³   ÚexponentÚmantissaÚcse_varsrR  Úcse_vars           rX   ÚfrexpÚCppOverrides.frexp�  s  € à˜a˜S Ð%¨°¨s°$Ð'7Ð7ˆ
ÜÑWÉJÓW×WÑWÜÑUÉ*ÓUÓUÐUä‹~ˆÜ—8‘8—<‘<×&Ñ&¬U¯[©[ÀÇÁÐ&ÐHˆÜ—8‘8—<‘<×&Ñ&¨Q¯W©W¸A¿G¹GÐ&ÐDˆØ�‰˜ ( ¨1Ð-Ô.Ø�‰˜˜x˜j¨°q°c¸¸X¸JÀbÐIÔJÜ	�‰×Ñ×Ñ Ô%ØÐ'ˆÜ"% jÖ";ÑˆIÜ�H‰H�L‰L×Ñ˜YÖ0ñ #<àÐ!Ð!rW   c                 ó   • SU  SU S3$ )Nzstd::hypot(rœ   r–   rV   r<  s     rX   ÚhypotÚCppOverrides.hypot®  r@  rW   c                 ó   • SU  S3$ )Nzstd::log10(r–   rV   rÁ  s    rX   Úlog10ÚCppOverrides.log10²  rÙ  rW   c                 ó   • SU  S3$ )Nz
std::log2(r–   rV   rÁ  s    rX   Úlog2ÚCppOverrides.log2¶  rÕ  rW   c                 ó   • SU  SU S3$ )Nzstd::ldexp(rœ   r–   rV   )r·  Úns     rX   ÚldexpÚCppOverrides.ldexpº  r@  rW   c                 ó   • SU  SU S3$ )Nzstd::nextafter(rœ   r–   rV   r<  s     rX   Ú	nextafterÚCppOverrides.nextafter¾  s   € à    2 a S¨Ð*Ð*rW   c                 ó¬   • [         R                  R                  nUS:X  a  gUS:X  a  U  S3$ US:X  a  U  SU  S3$ Uc	  SU  S	U  S
3$ [        SU< 35      e)NÚcompile_errorúcompile error!Úruntime_errorú	; throw 1rë  rì  rí  z	std::max(ú, decltype(ú)(0))ú7unrecognized config cpp.inject_relu_bug_TESTING_ONLY = ©r   rð  Úinject_relu_bug_TESTING_ONLYr†   rò  s     rX   ÚreluÚCppOverrides.reluÂ  s|   € ä�j‰j×5Ñ5ˆØ�/Ó!Ø#Ø�OÓ#Ø�S˜	�?Ð"Ø�JÓØ�S˜ Q C tÐ,Ð,Ø‰[Ø˜q˜c ¨Q¨C¨uÐ5Ð5ä ØIÈ#ÉÐQóð rW   c                 ó   • SU  SU S3$ )Nzmin_propagate_nan(rœ   r–   rV   r£  s     rX   ÚminimumÚCppOverrides.minimumÒ  ó   € à# A 3 b¨¨¨1Ð-Ð-rW   c                 ó   • SU  SU S3$ )Nzmax_propagate_nan(rœ   r–   rV   r£  s     rX   ÚmaximumÚCppOverrides.maximumÖ  r…  rW   c                 ó   • U  SU SU 3$ )Nú ? ú : rV   )r¤  r¥  Úcs      rX   ÚwhereÚCppOverrides.whereÚ  s   € à��C˜�s˜#˜a˜SÐ!Ð!rW   c                 ó   • SU  SU S3$ )Nzmod(rœ   r–   rV   r£  s     rX   r  ÚCppOverrides.modÞ  s   € à�a�S˜˜1˜#˜QÐÐrW   c                 ó(   • [        U [        U   5      $ r  )rP   rH   )Úvalrˆ   s     rX   ÚconstantÚCppOverrides.constantâ  s   € ä˜C¤¨eÑ!4Ó5Ð5rW   c                 ó  • [        [        R                  R                  U 5      5      n[        R                  R                  R                  [        R                  R                  U[        U 5      S9n[        R                  " X15      $ )N©Úbounds)
rD   r4   rr  Úrename_indexingr±  r²  r³  r&   r2   r¯  )r¹  rˆ   Úidx_strr§   s       rX   Ú
index_exprÚCppOverrides.index_expræ  s`   € äœŸ™×0Ñ0°Ó6Ó7ˆÜ�h‰h�l‰l×#Ñ#Ü�H‰H×Ñ˜gÔ.CÀDÓ.Ið $ð 
ˆô �|Š|˜CÓ'Ð'rW   c                 ó6  • [        5       n[        R                  R                  R	                  5       nUR                  SU S35        [        R                  R                  U5         UR                  5          U" 5       nUR                  SU S35        S S S 5        S S S 5        UR                  S5        [        R                  R                  R                  U5        [        USU S35      nU  SU SU 3$ ! , (       d  f       Nj= f! , (       d  f       Ns= f)	NrÕ   ú = [&]úreturn r»   rš  z())rŠ  z() : )r7   r4   rr  r±  rZ  rÂ   Úswap_buffersrÄ   r³  r\  rP   )ÚmaskÚbodyÚotherr³   Úbody_varr  Ú
other_codes          rX   ÚmaskedÚCppOverrides.maskedî  s×   € ä‹~ˆô —8‘8—<‘<×&Ñ&Ó(ˆØ�‰˜˜x˜j¨Ð/Ô0Ü�X‰X×"Ñ" 4Õ(¨$¯+©+­-Ù“VˆFØ�N‰N˜W V H¨AÐ.Ô/÷ +8×(ð 	�‰�sÔÜ	�‰×Ñ×Ñ Ô%ô " %¨9°X°J¸cÐ)BÓCˆ
Ø��s˜8˜* E¨*¨Ð6Ð6÷ +8­-ú×(Õ(ús$   Á'D
Á8C9ÂD
Ã9
D	ÄD
Ä

Dc                 ó   • U  SU 3$ )Nú && rV   r£  s     rX   Úlogical_andÚCppOverrides.logical_andÿ  ó   € à��D˜˜ˆ}ÐrW   c                 ó   • SU  3$ )NÚ!rV   ©r¤  s    rX   Úlogical_notÚCppOverrides.logical_not  ó   € à�1�#ˆwˆrW   c                 ó   • U  SU 3$ )Nr›   rV   r£  s     rX   Ú
logical_orÚCppOverrides.logical_or  r«  rW   c                 ó   • U  SU 3$ )Nú != rV   r£  s     rX   Úlogical_xorÚCppOverrides.logical_xor  r«  rW   c                 ó   • SU  SU  SU S3$ )Nrš  ú)(ú & r–   rV   r£  s     rX   Úbitwise_andÚCppOverrides.bitwise_and  ó   € à˜1˜#˜R ˜s # a S¨Ð*Ð*rW   c                 ó   • SU  SU  S3$ )Nrš  z)(~r–   rV   r®  s    rX   Úbitwise_notÚCppOverrides.bitwise_not  rÎ  rW   c                 ó   • SU  SU  SU S3$ )Nrš  rº  ú | r–   rV   r£  s     rX   Ú
bitwise_orÚCppOverrides.bitwise_or  r¾  rW   c                 ó   • SU  SU  SU S3$ )Nrš  rº  rš   r–   rV   r£  s     rX   Úbitwise_xorÚCppOverrides.bitwise_xor  r¾  rW   c                 óø  • [        5       nUR                  S5        UR                  5          [        U R                     nUR                  SU SU S35        UR                  SU SU SU S35        UR                  5          UR                  S	U  S
35        S S S 5        UR                  S	U  SU SU  SU S3	5        S S S 5        UR                  S5        U$ ! , (       d  f       NG= f! , (       d  f       N2= f)Nú[&]()úconstexpr decltype(ú) max_shift = sizeof(z) * CHAR_BIT;ú$if ((static_cast<std::make_signed_t<ú>>(ú) < 0) || (ú >= max_shift))úreturn decltype(z)(0);z#)(static_cast<std::make_unsigned_t<z) << rÖ   ú()©r7   rÂ   rÄ   rH   rˆ   ©r¤  r¥  r³   Úscalar_ts       rX   Úbitwise_left_shiftÚCppOverrides.bitwise_left_shift  sô   € ä‹~ˆØ�‰�wÔØ�[‰[�]Ü# A§G¡GÑ,ˆHØ�N‰NØ% a SÐ(=¸h¸ZÀ}ÐUôð �N‰NØ6°x°jÀÀAÀ3ÀkÐRSÐQTÐTcÐdôð —‘•Ø—‘Ð!1°!°°EÐ:Ô;÷ à�N‰NØ" 1 #Ð%HÈÈ
ÐRUÐVWÐUXÐX]Ð^_Ð]`Ð`bÐcô÷ ð 	�‰�tÔØˆ÷ •ú÷ �]ús$   ¬AC+ÂCÂ&C+Ã
C(	Ã$C+Ã+
C9c           
      óþ  • [        5       nUR                  S5        UR                  5          [        U R                     nUR                  SU SU SU S35        UR                  SU SU SU S	35        UR                  5          UR                  S
U  SU  S35        S S S 5        UR                  S
U  SU  SU S35        S S S 5        UR                  S5        U$ ! , (       d  f       ND= f! , (       d  f       N2= f)NrÊ  rË  rÌ  z ) * CHAR_BIT - std::is_signed_v<z>;rÍ  rÎ  rÏ  rÐ  rÑ  rº  z >> max_shift);ú >> rÖ   rÒ  rÓ  rÔ  s       rX   Úbitwise_right_shiftÚ CppOverrides.bitwise_right_shift3  sô   € ä‹~ˆØ�‰�wÔØ�[‰[�]Ü# A§G¡GÑ,ˆHØ�N‰NØ% a SÐ(=¸h¸ZÐGgÐhpÐgqÐqsÐtôð �N‰NØ6°x°jÀÀAÀ3ÀkÐRSÐQTÐTcÐdôð —‘•Ø—‘Ð!1°!°°B°q°c¸ÐIÔJ÷ à�N‰NÐ-¨a¨S°°1°#°T¸!¸¸BÐ?Ô@÷ ð 	�‰�tÔØˆ÷	 •ú÷ �]ús$   ¬AC.ÂCÂ#C.Ã
C+	Ã'C.Ã.
C<ÚseedrP  c                 ó   • SU  SU S3$ )Nznormalized_rand_cpu(rœ   r–   rV   ©rÜ  rP  s     rX   ÚrandÚCppOverrides.randE  s   € à% d V¨2¨f¨X°QÐ7Ð7rW   c                 ó   • SU  SU S3$ )Nz
randn_cpu(rœ   r–   rV   rÞ  s     rX   ÚrandnÚCppOverrides.randnI  s   € à˜D˜6  F 8¨1Ð-Ð-rW   c           	      ó    • SU  SU SU SU S3	$ )Nzrandint64_cpu(rœ   r–   rV   )rÜ  rP  ÚlowÚhighs       rX   Ú	randint64ÚCppOverrides.randint64M  s#   € à ˜v R¨ x¨r°#°°b¸¸¸aÐ@Ð@rW   c                 ó   • SU  SU  SU  S3$ )Nrš  z)(1) / (decltype(z)(1) + std::exp(-r¡   rV   rÁ  s    rX   ÚsigmoidÚCppOverrides.sigmoidQ  s   € à˜1˜#Ð.¨q¨cÐ1BÀ1À#ÀRÐHÐHrW   c           
      óX  • [        5       nSU  S3nSU  S3nUR                  S5        UR                  5          UR                  SU  SU SU S35        UR                  S	U  S
U SU S35        UR                  S5        S S S 5        UR                  S5        U$ ! , (       d  f       N!= f)Nrš  ú)(0)rí  rÊ  úauto left = z > 0 ? r‹  r»   úauto right = z < 0 ? úreturn left - right;rÒ  ©r7   rÂ   rÄ   )r·  r³   Úscalar_zeroÚ
scalar_ones       rX   ÚsignÚCppOverrides.signU  sª   € ä‹~ˆØ! !  DÐ)ˆØ    4Ð(ˆ
Ø�‰�wÔØ�[‰[�]Ø�N‰N˜\¨!¨¨G°J°<¸sÀ;À-ÈqÐQÔRØ�N‰N˜]¨1¨#¨W°Z°LÀÀKÀ=ÐPQÐRÔSØ�N‰NÐ1Ô2÷ ð 	�‰�tÔØˆ÷ �]ús   ¸ABÂ
B)rÇ   r‡   rï   Ú
extra_metar¸   c                 ó   • [         er  )ÚNotImplementedError)rA  rÇ   r‡   rï   rö  s        rX   Úpartial_accumulateÚCppOverrides.partial_accumulateb  s
   € ô "Ð!rW   rV   ©NT)Vr  r  r  r  r  Ústaticmethodr¦  rÊ   rc  r¯  r¾  rÂ  rÆ  rÉ  rÌ  rÐ  rÓ  r×  rÛ  rÞ  rá  rå  rè  rô  r÷  rú  rý  r   r  r  r	  r  r  r  r  r  r!  r$  r'  r*  r-  r0  r3  r6  r9  r>  rB  rE  rH  rd  rg  rj  rm  rq  rt  r€  rƒ  r‡  r�  r  r“  rš  r¥  r©  r¯  r³  r·  r¼  rÀ  rÄ  rÇ  rÖ  rÚ  rõ   ÚExprrß  râ  rç  rê  rô  rp   r9   Údictr   rù  r!  rV   rW   rX   rŸ  rŸ  ±  s+  † Ù%àñ7ó ð7ð ñ7ó ð7ð ñ7ó ð7ð ó(ó ð(ðT ñ<ó ð<ð ñ ó ð ð ñ ó ð ð ñ ó ð ð ñ&ó ð&ð ñ ó ð ð ñ!ó ð!ð ñ"ó ð"ð ñ ó ð ð ñ!ó ð!ð ñ#ó ð#ð ñ!ó ð!ð ñ%ó ð%ð ñ	ó ð	ð ñ ó ð ð ñ!ó ð!ð ñ	
ó ð	
ð ñ%ó ð%ð ñ ó ð ð ñ&ó ð&ð ñ"ó ð"ð ñYó ðYð ñ!ó ð!ð ñ"ó ð"ð ñó ðð ñ&ó ð&ð ñ"ó ð"ð ñ"ó ð"ð ñ#ó ð#ð ñ!ó ð!ð ñ"ó ð"ð ñ!ó ð!ð ñ!ó ð!ð ñ!ó ð!ð ñ"ó ð"ð ñ'ó ð'ð ñ!ó ð!ð ñ"ó ð"ð ñ*ó ð*ð ñ"ó ð"ð  ñ'ó ð'ð ñ"ó ð"ð ñ!ó ð!ð ñ'ó ð'ð ñ+ó ð+ð ñó ðð ñ.ó ð.ð ñ.ó ð.ð ñ"ó ð"ð ñ ó ð ð ñ6ó ð6ð ñ(ó ð(ð ñ7ó ð7ð  ñó ðð ñó ðð ñó ðð ñó ðð ñ+ó ð+ð ñ&ó ð&ð ñ+ó ð+ð ñ+ó ð+ð ñó ðð& ñó ðð" ð8�5—:‘:ð 8 u§z¡zó 8ó ð8ð ð.�E—J‘Jð .¨¯
©
ó .ó ð.ð ðA˜Ÿ
™
ð A¨E¯J©Jó Aó ðAð ñIó ðIð ñ
ó ð
ð"àð"ð ð"ð ð	"ð
 ˜˜c˜‘Nð"ð 
÷"rW   rŸ  rð  c                   ó   ^ • \ rS rSrSrU 4S jr\S 5       r\S 5       r\S 5       r	\S 5       r
\S 5       r\S	 5       r\S
 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r \S 5       r!\S 5       r"\S  5       r#\S! 5       r$\S" 5       r%\S# 5       r&\S$ 5       r'\S% 5       r(\S& 5       r)\S' 5       r*\S( 5       r+\S) 5       r,\S* 5       r-\S+ 5       r.\S, 5       r/\S- 5       r0\S. 5       r1\S/ 5       r2\S0 5       r3\S1 5       r4\S2 5       r5\S3 5       r6\S4 5       r7\S5 5       r8\S6 5       r9\S7 5       r:\S8 5       r;\S9 5       r<\S: 5       r=\S; 5       r>\S< 5       r?\S= 5       r@\S> 5       rA\S? 5       rB\S@ 5       rC\SA 5       rD\SB 5       rE\SC 5       rF\SD 5       rG\SE 5       rH\SF 5       rI\SG 5       rJ\SH 5       rK\SI 5       rL\SJ 5       rM\SK 5       rN\STSL j5       rO\SM 5       rP\SN 5       rQ\SO 5       rR\SP 5       rS\TSQ 5       rU\TSR 5       rVSSrWU =rX$ )UÚCppVecOverridesio  z.Map element-wise ops to aten vectorization C++c                 óö   >^• [         TU ]  U 5      mUU4S jn[        [        5      R	                  5        H?  u  pE[        USS 5      [        L d  M  US;  d  M#  [        TXC" UR                  5      5        MA     T$ )Nc                 ó   >^ • UU U4S jnU$ )Nc                  óÖ  >• U  Vs/ s HT  n[        U[        [        R                  45      (       d*  [        U[        5      (       d  M?  UR
                  (       a  MR  UPMV     nnU  Vs/ s H/  n[        U[        5      (       d  M  UR
                  (       d  M-  UPM1     nn[        U 5      nU(       aæ  U(       aß  / nU  H×  n[        U[        [        R                  45      (       až  [        U[        R                  5      (       a7  UR                  (       d&  [        R                  " U[        R                  5      nO%[        R                  " U[        R                  5      n[        U[        5      (       a  UR                  OUnUR                  U5        MÙ     U(       a?  [!        U5      S:X  a  [#        U5      nO$T
[$        R&                  L a  [#        USS  5      USS & U(       a¶  U(       a¯  [        [(        R*                  [,        5      (       d   eU Vs/ s H}  n[        U[        5      (       ac  UR
                  (       dR  T
[$        R.                  [$        R0                  [$        R2                  4;  a  [(        R*                  R5                  U5      OUPM     nnU(       a  T
" U0 UD6$ [7        [$        T5      n[9        UT
R:                  5      nUc   eU" U 0 UD6$ s  snf s  snf s  snf )Nr   r5   )r¢   rn   rõ   rý  rG   r¦   r5  Ú	is_numberr2   rš  r„   Úint64r“  r3   rï   ÚappendrÓ   rM   r   r�  r4   rr  ÚCppVecKernelrß  râ  rç  Ú	broadcastr?  rS  r  )r›  Úkwargsr˜  ÚscalarsÚvectorsÚnew_argsÚnew_argÚ
scalar_opsÚscalar_funcrD  ÚfuncrA  s            €€€rX   ÚwrapperÚ6CppVecOverrides.__new__.<locals>.wrap.<locals>.wrapper‚  s`  ø€ ñ  $óâ#˜Ü! #¬¬U¯Z©ZÐ'8×9Ñ9Ü" 3¬×7ó ð ADÇ
Å
÷ Ù#ð ð ñ  $óâ#˜Ü! #¤~×6ó à;>¿:½:÷ Ù#ð ð ô
   ›:�ÞžwØ!�HÛ#˜Ü% c¬C´·±Ð+<×=Ñ=Ü)¨#¬u¯z©z×:Ñ:À3Ç=Ç=Ü&)§n¢n°S¼%¿+¹+Ó&F¡ä&)§l¢l°3¼¿¹Ó&D Ü/9¸#¼x×/HÑ/H #§)¢)Èc˜CØ Ÿ™¨Ö,ñ  $ö ô ˜8“}¨Ó)Ü#/°Ó#9™Ø¤×!6Ñ!6Ò6Ü'3°H¸Q¸R°LÓ'A˜  ˜ö žwÜ%¤a§h¡h´×=Ñ=Ð=Ð=ñ  (0ó ò (0˜Gô !+¨7´N× CÑ CØ(/¯¯Ø$(ä$3×$8Ñ$8Ü$3×$9Ñ$9Ü$3×$=Ñ$=ð("ó%"ô	 ŸH™H×.Ñ.¨wÔ7ð ")ò)ñ (0ð ð  ö$ Ù Ð4¨VÑ4Ð4ô "'¤¸Ó!=�JÜ")¨*°d·m±mÓ"D�KØ&Ñ2Ð2Ð2Ù&¨Ð7°Ñ7Ð7ùòùòùò@ s*   †>KÁKÁKÁ'K!ÂK!ÂK!ÈBK&rV   )r  r  rD  rA  s   ` €€rX   ÚwrapÚ%CppVecOverrides.__new__.<locals>.wrapu  s   ù€ ÷@8ðD ˆNrW   rD  )r¥  rš  )	r?  Ú__new__Úvarsr   ÚitemsrS  rü  ÚsetattrÚ__func__)r7  r›  Úkargsr  rÇ   ÚmethodrA  rD  s         @€rX   r  ÚCppVecOverrides.__new__r  sq   ù€ Ü‰w‰˜sÓ#ˆöO	ôb !¤Ó1×7Ñ7Ö9‰LˆDÜ�v˜{¨DÓ1´\ÔAÀdð Sõ Gô ˜˜d D¨¯©Ó$9Ö:ñ :ð ˆrW   c                 ó   • U  SU 3$ )Nré   rV   r£  s     rX   r¦  ÚCppVecOverrides.addÏ  ó   € à��C˜�sˆ|ÐrW   c                 ó   • U  SU 3$ )Nrª  rV   r£  s     rX   rÊ   ÚCppVecOverrides.subÓ  r  rW   c                 ó   • U  SU 3$ ©Nr™   rV   r£  s     rX   rc  ÚCppVecOverrides.mul×  r  rW   c                 ó   • U  SU 3$ r  rV   r£  s     rX   ÚtruedivÚCppVecOverrides.truedivÛ  r  rW   c                 ó   • U  S3$ )Nz.abs()rV   rÁ  s    rX   rÂ  ÚCppVecOverrides.absß  ó   € à��Fˆ|ÐrW   c                 ó   • U  S3$ )Nz.sin()rV   rÁ  s    rX   rÆ  ÚCppVecOverrides.sinã  r*  rW   c                 ó   • U  S3$ )Nz.cos()rV   rÁ  s    rX   rÉ  ÚCppVecOverrides.cosç  r*  rW   c                 ó   • U  S3$ )Nz.exp()rV   rÁ  s    rX   rÐ  ÚCppVecOverrides.expë  r*  rW   c                 ó   • U  S3$ )Nz.exp2()rV   rÁ  s    rX   rÓ  ÚCppVecOverrides.exp2ï  ó   € à��Gˆ}ÐrW   c                 ó   • SU  S3nU  SU 3$ )Nrš  rí  z	.exp() - rV   )r·  Úvec_ones     rX   r×  ÚCppVecOverrides.expm1ó  s#   € ð ˜a˜S Ð%ˆØ��I˜g˜YÐ'Ð'rW   c                 ó   • U  S3$ )Nz.erf()rV   rÁ  s    rX   rÛ  ÚCppVecOverrides.erfù  r*  rW   c                 ó   • U  S3$ )Nz.erfc()rV   rÁ  s    rX   rÞ  ÚCppVecOverrides.erfcý  r3  rW   c                 ó   • U  S3$ )Nz	.erfinv()rV   rÁ  s    rX   rá  ÚCppVecOverrides.erfinv  ó   € à��IˆÐrW   c                 ó   • U  S3$ )Nz.sqrt()rV   rÁ  s    rX   rå  ÚCppVecOverrides.sqrt  r3  rW   c                 óü   • [        [        R                  [        5      (       d   e[        U [        5      (       d   eU R
                  c   e[        R                  R                  U R
                  5       SU  SU S3$ )Nr¡  ú == r–   ©r¢   r4   rr  r  rG   rˆ   Ú_get_mask_typer<  s     rX   ÚeqÚCppVecOverrides.eq	  ói   € äœ!Ÿ(™(¤L×1Ñ1Ð1Ð1Ü˜!œ^×,Ñ,Ð,Ð,Ø�w‰wÑ"Ð"Ð"Ü—(‘(×)Ñ)¨!¯'©'Ó2Ð3°1°Q°C°t¸A¸3¸aÐ@Ð@rW   c                 óÔ  • [        [        R                  [        5      (       d   e[        U [        5      (       d   eU R
                  [        R                  :X  aN  UR
                  [        R                  :X  d   e[        [        R                  R                  X45      u  p#U SU 3$ U R
                  c   e[        R                  R                  U R
                  5       SU  SU S3$ )Nr¶  r¡  r–   )r¢   r4   rr  r  rG   rˆ   r„   rl   rO   r³  rC  )r·  r=  Úx_castÚy_casts       rX   ÚneÚCppVecOverrides.ne  sº   € äœ!Ÿ(™(¤L×1Ñ1Ð1Ð1Ü˜!œ^×,Ñ,Ð,Ð,Ø�7‰7”e—j‘jÓ Ø—7‘7œeŸj™jÓ(Ð(Ð(Ü1´!·(±(×2BÑ2BÀQÀFÓK‰NˆFØ�X˜T & Ð*Ð*à—7‘7Ñ&Ð&Ð&Ü—h‘h×-Ñ-¨a¯g©gÓ6Ð7°q¸¸¸4À¸sÀ!ÐDÐDrW   c                 óü   • [        [        R                  [        5      (       d   e[        U [        5      (       d   eU R
                  c   e[        R                  R                  U R
                  5       SU  SU S3$ )Nr¡  r¼   r–   rB  r<  s     rX   ÚltÚCppVecOverrides.lt  ói   € äœ!Ÿ(™(¤L×1Ñ1Ð1Ð1Ü˜!œ^×,Ñ,Ð,Ð,Ø�w‰wÑ"Ð"Ð"Ü—(‘(×)Ñ)¨!¯'©'Ó2Ð3°1°Q°C°s¸1¸#¸QÐ?Ð?rW   c                 óü   • [        [        R                  [        5      (       d   e[        U [        5      (       d   eU R
                  c   e[        R                  R                  U R
                  5       SU  SU S3$ )Nr¡  z > r–   rB  r<  s     rX   ÚgtÚCppVecOverrides.gt#  rO  rW   c                 óü   • [        [        R                  [        5      (       d   e[        U [        5      (       d   eU R
                  c   e[        R                  R                  U R
                  5       SU  SU S3$ )Nr¡  ú <= r–   rB  r<  s     rX   ÚleÚCppVecOverrides.le*  rF  rW   c                 óü   • [        [        R                  [        5      (       d   e[        U [        5      (       d   eU R
                  c   e[        R                  R                  U R
                  5       SU  SU S3$ )Nr¡  ú >= r–   rB  r<  s     rX   ÚgeÚCppVecOverrides.ge1  rF  rW   c                 ó   • U  SU 3$ ©Nr»  rV   r<  s     rX   Úand_ÚCppVecOverrides.and_8  r  rW   c                 ó   • U  S3$ )Nz.rsqrt()rV   rÁ  s    rX   rè  ÚCppVecOverrides.rsqrt<  ó   € à��Hˆ~ÐrW   c                 ó   • U  SU S3$ )Nz.pow(r–   rV   r£  s     rX   r   ÚCppVecOverrides.pow@  s   € à��E˜!˜˜AˆÐrW   c                 ó   • U  S3$ )Nz.log()rV   rÁ  s    rX   r  ÚCppVecOverrides.logD  r*  rW   c                 ó   • U  S3$ )Nz.round()rV   rÁ  s    rX   r  ÚCppVecOverrides.roundH  ra  rW   c                 ó   • U  S3$ )Nz.floor()rV   rÁ  s    rX   r	  ÚCppVecOverrides.floorL  ra  rW   c                 ó   • U  S3$ )Nz.ceil()rV   rÁ  s    rX   r  ÚCppVecOverrides.ceilP  r3  rW   c                 ó   • U  S3$ )Nz.trunc()rV   rÁ  s    rX   r  ÚCppVecOverrides.truncT  ra  rW   c                 ó   • U  SU S3$ )Nz.fmod(r–   rV   r£  s     rX   r  ÚCppVecOverrides.fmodX  s   € à��F˜1˜#˜QÐÐrW   c                 ó   • U  S3$ )Nz	.lgamma()rV   rÁ  s    rX   r'  ÚCppVecOverrides.lgamma\  r=  rW   c                 ó*   • [        X5      u  pU  SU 3$ r\  ©rL   r£  s     rX   r©  ÚCppVecOverrides.logical_and`  ó   € ä,¨QÓ2‰ˆØ��C˜�sˆ|ÐrW   c                 ó   • SU  3$ ©NÚ~rV   r®  s    rX   r¯  ÚCppVecOverrides.logical_note  r±  rW   c                 ó*   • [        X5      u  pU  SU 3$ ©NrÃ  rs  r£  s     rX   r³  ÚCppVecOverrides.logical_ori  ru  rW   c                 ó*   • [        X5      u  pU  SU 3$ ©Nrš   rs  r£  s     rX   r·  ÚCppVecOverrides.logical_xorn  ru  rW   c                 ó*   • [        X5      u  pU  SU 3$ r\  rs  r£  s     rX   r¼  ÚCppVecOverrides.bitwise_ands  ru  rW   c                 ó   • SU  3$ rw  rV   r®  s    rX   rÀ  ÚCppVecOverrides.bitwise_notx  r±  rW   c                 ó*   • [        X5      u  pU  SU 3$ r{  rs  r£  s     rX   rÄ  ÚCppVecOverrides.bitwise_or|  ru  rW   c                 ó*   • [        X5      u  pU  SU 3$ r~  rs  r£  s     rX   rÇ  ÚCppVecOverrides.bitwise_xor�  ru  rW   c                 ó   • U  SU 3$ )Nz << rV   r£  s     rX   rÖ  Ú"CppVecOverrides.bitwise_left_shift†  r«  rW   c                 ó   • U  SU 3$ )NrÙ  rV   r£  s     rX   rÚ  Ú#CppVecOverrides.bitwise_right_shiftŠ  r«  rW   c                 óŒ   • [        [        R                  [        5      (       d   e[        R                  R	                  X5       $ r  )r¢   r4   rr  r  Úload)rÇ   rP  s     rX   Ú	load_seedÚCppVecOverrides.load_seedŽ  s/   € äœ!Ÿ(™(¤L×1Ñ1Ð1Ð1Ü—(‘(—-‘- Ó-Ð.Ð/rW   c                 ó„   • [        [        R                  [        5      (       d   e[	        5       nSU  S3n[        XU5      $ )Nz)result[offset_idx] = normalized_rand_cpu(ú, offset[offset_idx]);©r¢   r4   rr  r  r7   rF   ©rÜ  rP  r³   Úrand_functions       rX   rß  ÚCppVecOverrides.rand“  sA   € äœ!Ÿ(™(¤L×1Ñ1Ð1Ð1Ü‹~ˆà7¸°vÐ=SÐTð 	ô ˜F¨-Ó8Ð8rW   c                 ó„   • [        [        R                  [        5      (       d   e[	        5       nSU  S3n[        XU5      $ )Nzresult[offset_idx] = randn_cpu(r‘  r’  r“  s       rX   râ  ÚCppVecOverrides.randnœ  s<   € äœ!Ÿ(™(¤L×1Ñ1Ð1Ð1Ü‹~ˆØ9¸$¸Ð?UÐVˆÜ˜F¨-Ó8Ð8rW   c                 ó®   • [        [        R                  [        5      (       d   e[	        5       nSU  SU SU S3n[        XU[        R                  5      $ )Nz#result[offset_idx] = randint64_cpu(z, offset[offset_idx], rœ   rÖ   )r¢   r4   rr  r  r7   rF   r„   r  )rÜ  rP  rå  ræ  r³   r”  s         rX   rç  ÚCppVecOverrides.randint64£  sT   € äœ!Ÿ(™(¤L×1Ñ1Ð1Ð1Ü‹~ˆØ=¸d¸VÐCYÐZ]ÐY^Ð^`ÐaeÐ`fÐfhÐiˆÜ˜F¨-¼¿¹ÓEÐErW   c                 ó~   • U R                   UR                   :X  d   S5       eU  S[        R                  X5       SU 3$ )Nz;remainder vec implementation expect the same inputs' dtype.z - (z) * )rˆ   r   r  r£  s     rX   Ú	remainderÚCppVecOverrides.remainderª  sG   € à�w‰w˜!Ÿ'™'Ó!ð 	
ØIó	
Ð!ð ��Dœ×1Ñ1°!Ó7Ð8¸¸Q¸CÐ@Ð@rW   c                 ó   • U  S3$ )Nz.tan()rV   r®  s    rX   r÷  ÚCppVecOverrides.tan±  r*  rW   c           	      óŽ   • [         R                  R                  (       a"  SU  S3nSU  S3nSU  S3nU SU SU SU  SU 3	$ U  S	3$ )
Nrš  rí  z)(2)z)(-2)z / (z + (r™   z).exp()) - z.tanh())r   rð  Úuse_decompose_tanh)r¤  r5  Úvec_twoÚvec_minus_twos       rX   rú  ÚCppVecOverrides.tanhµ  sl   € ä�:‰:×(×(Ø! !  DÐ)ˆGØ! !  DÐ)ˆGØ'¨ s¨%Ð0ˆMà�)˜4 ˜y¨¨]¨O¸3¸q¸cÀÈWÈIÐVðð �S˜�=Ð rW   c                 ó   • U  S3$ )Nz.reciprocal()rV   r®  s    rX   Ú
reciprocalÚCppVecOverrides.reciprocalÁ  s   € à��MÐ"Ð"rW   c                 ó   • U  S3$ )Nz.atan()rV   rÁ  s    rX   rB  ÚCppVecOverrides.atanÅ  r3  rW   c                 ó   • U  S3$ )Nz.acos()rV   rÁ  s    rX   r*  ÚCppVecOverrides.acosÉ  r3  rW   c                 ó   • U  S3$ )Nz.asin()rV   rÁ  s    rX   r6  ÚCppVecOverrides.asinÍ  r3  rW   c                 ó   • U  S3$ )Nz.cosh()rV   rÁ  s    rX   r0  ÚCppVecOverrides.coshÑ  r3  rW   c                 ó   • U  S3$ )Nz.sinh()rV   rÁ  s    rX   r3  ÚCppVecOverrides.sinhÕ  r3  rW   c                 ó   • U  S3$ )Nz.log10()rV   rÁ  s    rX   rj  ÚCppVecOverrides.log10Ù  ra  rW   c                 ó   • U  S3$ )Nz.log2()rV   rÁ  s    rX   rm  ÚCppVecOverrides.log2Ý  r3  rW   c                 ó   • U  SU S3$ )Nz.nextafter(r–   rV   r<  s     rX   rt  ÚCppVecOverrides.nextafterá  s   € à��K ˜s !Ð$Ð$rW   c                 ó   • U  SU S3$ )Nz
.copysign(r–   rV   r£  s     rX   rH  ÚCppVecOverrides.copysignå  s   € à��J˜q˜c Ð#Ð#rW   c                 ó   • U  SU S3$ )Nz.atan2(r–   rV   r£  s     rX   r>  ÚCppVecOverrides.atan2é  ó   € à��G˜A˜3˜aÐ Ð rW   c                 ó   • U  SU S3$ )Nz.hypot(r–   rV   r£  s     rX   rg  ÚCppVecOverrides.hypotí  r»  rW   c           
      ó<   • SU  S3nSU  S3nU SU SU  SU SU  S3
$ )	Nrš  rí  z)(0.5)z * ((ré   z)/(rª  z)).log()rV   )r·  r5  Úvec_one_halfs      rX   rE  ÚCppVecOverrides.atanhñ  sE   € ð ˜a˜S Ð%ˆØ" 1 # VÐ,ˆØ�˜u W I¨S°°°3°w°i¸sÀ1À#ÀXÐNÐNrW   c                 ó   • U  S3$ )Nz.asinh()rV   rÁ  s    rX   r9  ÚCppVecOverrides.asinhø  ra  rW   c                 ó   • U  S3$ )Nz.acosh()rV   rÁ  s    rX   r-  ÚCppVecOverrides.acoshü  ra  rW   c                 ó¬   • [         R                  R                  nUS:X  a  gUS:X  a  U  S3$ US:X  a  U  SU  S3$ Uc	  SU  S	U  S
3$ [        SU< 35      e)Nrw  rx  ry  rz  rë  rì  rí  zat::vec::clamp_min(r{  r|  r}  r~  rò  s     rX   r€  ÚCppVecOverrides.relu   s|   € ä�j‰j×5Ñ5ˆØ�/Ó!Ø#Ø�OÓ#Ø�S˜	�?Ð"Ø�JÓØ�S˜ Q C tÐ,Ð,Ø‰[Ø(¨¨¨;°q°c¸Ð?Ð?ä ØIÈ#ÉÐQóð rW   c                 ó   • SU  SU  SU  S3$ )Nrš  z)(1)/(decltype(z)(1) + z.neg().exp())rV   rÁ  s    rX   rê  ÚCppVecOverrides.sigmoid  s   € à˜1˜#˜_¨Q¨C¨w°q°c¸ÐGÐGrW   c                 ó   • U  S3$ )Nz.neg()rV   rÁ  s    rX   rÌ  ÚCppVecOverrides.neg  r*  rW   c                 óè  • [        U R                  5      (       a*  U R                  UR                  :X  d   S5       eSU  SU S3$ [        S X4 5       5      (       d   eSU  S3n[        R                  R                  UR                  5      S:  a,  U SS[        R                  R                  -  S-
   S	U S
U S3nU  SU 3nSU  SU SU S3nSU  SU SU SU S3	nU SU SU SU S
U SU S3$ )NzDdiv_floor_floating_vec implementation expect the same inputs' dtype.zdiv_floor_floating_vec(rœ   r–   c              3   óL   #   • U  H  n[        UR                  5      v •  M     g 7fr  )r   rˆ   )r*  Úitems     rX   r,  Ú+CppVecOverrides.floordiv.<locals>.<genexpr>!  s   é € ÐGÂ¸Ô'¨¯
©
×3Ð3Âùó   ‚"$rš  r5   ú::blend<r½  ú(1), r  r¡  r  r¶  z(0))r  r¼   z	(0)) != (z(0)))z	::blendv(rª  r»  )r   rˆ   r4  r4   rr  Ú_get_raw_num_vectorsÚtiling_factor)r¤  r¥  Ú_tr  Úhas_remÚis_negs         rX   r  ÚCppVecOverrides.floordiv  s5  € ä˜!Ÿ'™'×"Ñ"Ø—7‘7˜aŸg™gÓ%ð ØVóÐ%ð -¨Q¨C¨r°!°°AÐ6Ð6äÑGÀÁÓG×GÑGÐGÐGà˜Q˜C˜qÐ!ˆBÜ�x‰x×,Ñ,¨Q¯W©WÓ5¸Ó9à�d˜( A¬¯©×)?Ñ)?Ñ$?À1Ñ#DÐ"EÀRÈÀtÈ5ÐQRÐPSÐSTÐU�Ø�S˜˜A˜3�<ˆDØ˜!˜˜C ˜s $ r d¨$Ð/ˆGØ˜!˜˜C ˜t 9¨Q¨C¨s°2°$°eÐ<ˆFØ�T˜ 4 &¨¨4¨&°°B°4°u¸W¸IÀSÈÈÐPQÐRÐRrW   c                 óÎ   • [         R                  R                  UR                  5      S:  a2  SU S3nU SS[         R                  R                  -  S-
   SU SU S3nU  SU 3$ )Nr5   rš  r–   rÐ  r½  rÑ  r  )r4   rr  rÒ  rˆ   rÓ  )r¤  r¥  rÔ  s      rX   r  ÚCppVecOverrides.truncdiv,  sp   € ô �8‰8×(Ñ(¨¯©Ó1°AÓ5à˜Q˜C˜qÐ!ˆBØ�$�h ¤Q§X¡X×%;Ñ%;Ñ ;¸qÑ@ÐAÀÀBÀ4ÀuÈQÈCÈqÐQˆAØ��C˜�sˆ|ÐrW   c                 óì   • U R                   [        R                  :X  aN  UR                   [        R                  :X  d   e[        [        R
                  R                  X45      u  p#U SU 3$ SU  SU S3$ )Nr»  úat::vec::minimum(rœ   r–   ©rˆ   r„   rl   rO   r4   rr  r³  ©r¤  r¥  Úa_castÚb_casts       rX   rƒ  ÚCppVecOverrides.minimum5  óh   € à�7‰7”e—j‘jÓ Ø—7‘7œeŸj™jÓ(Ð(Ð(Ü1´!·(±(×2BÑ2BÀQÀFÓK‰NˆFØ�X˜S  Ð)Ð)à& q c¨¨A¨3¨aÐ0Ð0rW   c                 óì   • U R                   [        R                  :X  aN  UR                   [        R                  :X  d   e[        [        R
                  R                  X45      u  p#U SU 3$ SU  SU S3$ )NrÃ  úat::vec::maximum(rœ   r–   rÜ  rÝ  s       rX   r‡  ÚCppVecOverrides.maximum>  rá  rW   c                 ó   • U  SU  3$ r#  rV   r®  s    rX   ÚsquareÚCppVecOverrides.squareG  r  rW   c                 ó¤  • [        [        R                  [        5      (       d   eUR                  [
        R                  :X  aX  UR                  [
        R                  :X  d   e[        [        R                  R                  XU45      u  p4nSU SU SU SU S3	$ SU SU SU S[        R                  R                  XR                  5       S3	$ )Nrš  ú
)::blendv(rœ   r–   )
r¢   r4   rr  r  rˆ   r„   rl   rO   r³  Ú_get_mask_cast)r¤  r¥  rŒ  Úblendv_aÚblendv_bÚblendv_cs         rX   r�  ÚCppVecOverrides.whereK  s½   € äœ!Ÿ(™(¤L×1Ñ1Ð1Ð1Ø�7‰7”e—j‘jÓ Ø—7‘7œeŸj™jÓ(Ð(Ð(Ü+?Ü—‘× Ñ  1¨ )ó,Ñ(ˆH ð ˜x˜j¨
°8°*¸B¸x¸jÈÈ8È*ÐTUÐVÐVà˜q˜c ¨A¨3¨b°°°2´a·h±h×6MÑ6MÈa×QXÑQXÓ6YÐ5ZÐZ[Ð\Ð\rW   c                 óˆ  • [        5       nSU  S3nSU  S3nSU  SU SU SU SU  S3nSU  SU SU SU  SU S3nUR                  S5        UR                  5          UR                  S	U S
35        UR                  SU S
35        UR                  S5        S S S 5        UR                  S5        U$ ! , (       d  f       N!= f)Nrš  rí  rí  ré  rœ   r¼   r–   rÊ  rî  r»   rï  rð  rÒ  rñ  )r·  r³   Úvec_zeror5  Úblendv_lÚblendv_rs         rX   rô  ÚCppVecOverrides.signW  sà   € ä‹~ˆØ˜q˜c Ð&ˆØ˜a˜S Ð%ˆØ˜q˜c ¨H¨:°R¸°yÀÀ8À*ÈCÐPQÈsÐRSÐTˆØ˜q˜c ¨H¨:°R¸°yÀÀ1À#ÀSÈÈ
ÐRSÐTˆØ�‰�wÔØ�[‰[�]Ø�N‰N˜\¨(¨°1Ð5Ô6Ø�N‰N˜]¨8¨*°AÐ6Ô7Ø�N‰NÐ1Ô2÷ ð 	�‰�tÔØˆ÷ �]ús   Á<B3Â3
Cc                 ó.  • U[         R                  [         R                  [         R                  [         R                  [         R
                  [         R                  [         R                  [         R                  [         R                  [         R                  [         R                  4;   d   [         SU 35       e[        U [        5      (       d   eU R                  n[         R"                  R%                  XU5      n[         R"                  R&                  R)                  [         R"                  R*                  U5      nUR-                  SX4SU05        U[.        ;   a4  U[         R                  :X  a   [         R"                  R1                  XXQ5        U$ )Nz does not support r¯  rª   )r„   rl   Úfloat64ro   Úbfloat16Úfloat16Úuint8Úint8r[  r  Úfloat8_e4m3fnÚfloat8_e5m2r  r¢   rG   rˆ   r4   rr  r°  r±  r²  r³  r´  rƒ   rµ  r¶  s         rX   r¯  ÚCppVecOverrides.to_dtypef  s  € àÜ�J‰JÜ�M‰MÜ�K‰KÜ�N‰NÜ�M‰MÜ�K‰KÜ�J‰JÜ�K‰KÜ�K‰KÜ×ÑÜ×Ñð
ó 
ð 	2ô ˆZÐ)¨%¨Ð1ó	2ð 
ô ˜!œ^×,Ñ,Ð,Ð,Ø—G‘Gˆ	Ü�x‰x×)Ñ)¨!°IÓ>ˆÜ—‘—‘×&Ñ&¤q§x¡x×'7Ñ'7¸Ó>ˆØ×Ñ˜j¨1¨*°{ÀIÐ6NÔOØ”MÓ! i´5·;±;Ó&>Ü�H‰H×(Ñ(¨°vÔEØˆrW   c                 ó€   • [         R                  R                  nUS:X  a  U  SU  S3$ Uc  U  S3$ [        SU< 35      e)Nrë  rì  rí  z.log1p()rî  rï  rò  s     rX   rô  ÚCppVecOverrides.log1p~  sT   € ä�j‰j×6Ñ6ˆØ�*ÓØ�S˜ Q C tÐ,Ð,Ø‰[Ø�S˜�>Ð!ä ØJÈ3É'ÐRóð rW   c                 ó"
  ^• [        [        R                  [        5      (       d   e[	        5       n[        R                  R
                  R                  5       n[        R                  R                  U 5       nUR                  SU S35        [        R                  R                  U5         UR                  5          U" 5       nUR                  SU S35        S S S 5        S S S 5        S S S 5        UR                  S5        [        R                  R                  R                  U5        WR                  mU S3nU4S jnUR                  (       a  Un	OU" U5      n	[        U[         T   5      n
U" U
5      n[        W["        5      (       d   U5       eUR                  (       Ga-  [	        5       nUR                  S5        [        R                  R                  U5         UR                  5          UR                  SU S	35        UR                  5          UR                  SU S35        S S S 5        UR                  S
5        UR                  5          [        R                  R
                  R%                  [        R                  R                  U	5      n[        R                  R
                  R%                  [        R                  R                  U5      n[        U["        5      (       d   U5       e[        U["        5      (       d   U5       eTUl        TUl        [        R                  R&                  nUR                  SUR)                  X\U5       S35        S S S 5        S S S 5        S S S 5        UR                  S5        [        R                  R
                  R%                  [        R                  R                  U5      nSUl        O¦UR                  (       aK  [        R                  R
                  R%                  [        R                  R                  U  SU	 SU 35      nOJ[        R                  R
                  R%                  [        R                  R                  U  SU SU
 35      nUR+                  SXX&40 5        U$ ! , (       d  f       GN¿= f! , (       d  f       GNÉ= f! , (       d  f       GNÓ= f! , (       d  f       GN = f! , (       d  f       GN= f! , (       d  f       GN‰= f! , (       d  f       GN“= f)NrÕ   r�  rž  r»   rÒ  c                 ó¾   >• T[         R                  :X  a$  [        R                  R	                  5        SU  S3$ [        R                  R                  T5       SU  S3$ )Nú::from(r–   r¡  )r„   rl   r4   rr  rC  Ú_get_vec_type)r³   rˆ   s    €rX   Úmaskify_or_vecifyÚ1CppVecOverrides.masked.<locals>.maskify_or_vecifyš  s]   ø€ ð œEŸJ™JÓ&ô —8‘8×*Ñ*Ó,Ð-¨W°T°F¸!Ð<ðô Ÿ™×.Ñ.¨uÓ5Ð6°a¸°v¸QÐ?ðrW   ú[&]úif (z.all_zero())ÚelseTrŠ  r‹  r¥  )r¢   r4   rr  r  r7   r±  rZ  r¥  rÂ   rŸ  rÄ   r³  r\  rˆ   r¦   rP   rH   rG   r²  Ú	overridesr�  r´  )r   r¡  r¢  r³   r§   Únew_maskr  Ú	body_coder  Úbody_code_vecr¤  Úother_code_vecÚbody_vec_varÚother_vec_varr  rº  rˆ   s                   @rX   r¥  ÚCppVecOverrides.maskedŠ  sš  ø€ äœ!Ÿ(™(¤L×1Ñ1Ð1Ð1Ü‹~ˆÜ�h‰h�l‰l×!Ñ!Ó#ˆÜ�X‰X�_‰_˜TÔ" hØ�N‰N˜U 3 % vÐ.Ô/Ü—‘×&Ñ& tÕ,¨d¯k©k­mÙ›�Ø—‘ ¨¨°Ð2Ô3÷ /<×,÷ #ð
 	�‰�sÔÜ	�‰×Ñ×Ñ Ô%à—‘ˆØ�e˜2�Jˆ	õ	ð �=�=Ø%‰Má-¨iÓ8ˆMÜ! %¬°eÑ)<Ó=ˆ
á*¨:Ó6ˆÜ˜(¤N×3Ñ3Ð=°XÓ=Ð3Ø�?�?ˆ?Ü“>ˆDØ�N‰N˜5Ô!Ü—‘×&Ñ& tÕ,¨d¯k©k­mØ—‘  h Z¨|Ð<Ô=Ø—[‘[•]Ø—N‘N W¨^Ð,<¸AÐ#>Ô?÷ #à—‘˜vÔ&Ø—[‘[•]ä#$§8¡8§<¡<×#8Ñ#8ÜŸ™×(Ñ(Ø%ó$�Lô %&§H¡H§L¡L×$9Ñ$9ÜŸ™×(Ñ(Ø&ó%�Mô & l´N×CÑCÐQÀ\ÓQÐCÜ% m´^×DÑDÐSÀmÓSÐDØ).�LÔ&Ø*/�MÔ'ô Ÿ™×*Ñ*ð ð —N‘NØ! )§/¡/°(È-Ó"XÐ!YÐYZÐ[ô÷% #÷ /<×,ð4 �N‰N˜4Ô Ü—X‘X—\‘\×*Ñ*Ü—‘× Ñ ØóˆFð !ˆF�MØ�]�]Ü—X‘X—\‘\×*Ñ*Ü—‘× Ñ  T F¨#¨m¨_¸CÀÐ?OÐ"Pó‰Fô —X‘X—\‘\×*Ñ*Ü—‘× Ñ  T F¨#¨i¨[¸¸J¸<Ð"HóˆFð
 	×Ñ˜h¨°UÐ(CÀRÔHØˆ÷M /<®mú×,Ö,ú÷ #Ö"ú÷> #–]ú÷ #–]ú÷ /<®mú×,Ö,ús‘   Á85R7Â-R%Â>RÃR%Ã#R7Ç#S?Ç4&S-ÈS	È0)S-ÉD
SÍ#S-Í+S?Ò
R"ÒR%Ò%
R4	Ò/R7Ò7
SÓ	
SÓS-Ó
S*Ó%S-Ó-
S<	Ó7S?Ó?
Tc                 ó`  • [        [        R                  [        5      (       d   e[        R                  R	                  U 5      n[        R                  R
                  [        R                  R                     n[        R                  R                  X#5      nUS:X  a  [        R                  X5      $ Ubª  [        R                  R                  R                  [        R                  R                  [        U5      [        U 5      S9n[        R                   " XQ5      n[        U["        5      (       a  UR$                  n[        R                  R'                  Xd5      nO9[        R                  R)                  S X![        R                  R                  5      nUR+                  SX40 5        U$ )Nr   r–  rš  )r¢   r4   rr  r  r˜  ÚitervarsÚ
tiling_idxÚ_try_get_const_striderŸ  rš  r±  r²  r³  rD   r&   r2   r¯  r3   rï   ÚarangeÚ_load_or_store_non_contiguousr´  )r¹  rˆ   r’   Ú
tiling_varÚstriderg  rï   rº  s           rX   rš  ÚCppVecOverrides.index_exprÙ  s-  € äœ!Ÿ(™(¤L×1Ñ1Ð1Ð1Ü—‘×(Ñ(¨Ó.ˆÜ—X‘X×&Ñ&¤q§x¡x×':Ñ':Ñ;ˆ
Ü—‘×/Ñ/°ÓBˆØ�Q‹;Ü×*Ñ*¨4Ó7Ð7ØÑÜ—(‘(—,‘,×'Ñ'Ü—‘× Ñ ¤%¨£,Ô7LÈTÓ7Rð (ð ˆCô —L’L Ó,ˆEÜ˜%¤×*Ñ*ØŸ™�Ü—X‘X—_‘_ UÓ3‰Fä—X‘X×;Ñ;Ø�e¤A§H¡H×$4Ñ$4óˆFð 	×Ñ˜l¨T¨M¸2Ô>ØˆrW   c           
      óˆ  • SU  S3SU  S34n[        S U 5       5      (       a  [        S U 5       5      $ [        U R                     n[        R
                  R                  (       a  [        R
                  R                  O[        R
                  R                  n[        5       n[        R
                  R                  R                  [        R                  S9n[        R
                  R                  R                  U R                  S9nUR                  SU 40 S9  UR                  SU 40 S9  [        R
                  R                  U R                  5      nUS	:X  a  S
U S3OSU SU S3nUR                  US	:X  a  SU S3OSU SU S35        UR                  U SU S35        UR                  S5        UR!                  5          UR                  SU S[        R
                  R                   S35        UR                  U  S[#        U5       S35        UR                  S[        R
                  R                   S35        UR                  SU S[        R
                  R                   S35        UR                  S[#        U5       S35        UR!                  5          UR                  S5        S S S 5        UR                  US	:X  a  U S[#        U5       S3OU SU S [#        U5       S35        UR                  U S!U S"[#        U5       S35        S S S 5        UR                  S#5        [        R
                  R$                  R'                  U5        Xe4n	[)        X5       H.  u  p«[        R
                  R                  R+                  X«5        M0     Xe4$ ! , (       d  f       Nð= f! , (       d  f       Nœ= f)$NrK  rL  rM  c              3   óx   #   • U  H0  n[         R                  R                  R                  U5      S Lv •  M2     g 7fr  rO  rQ  s     rX   r,  Ú(CppVecOverrides.frexp.<locals>.<genexpr>ô  rT  rU  c              3   ót   #   • U  H.  n[         R                  R                  R                  U5      v •  M0     g 7fr  rO  rQ  s     rX   r,  r  õ  rW  rX  ©rˆ   rd  )r	  r5   úat::vec::Vectorized<r�   úat::vec::VectorizedN<rœ   zat::vec::Vectorized<int32_t> r»   zat::vec::VectorizedN<int32_t, ú> r˜   rÊ  ú__at_align__ std::array<ú	> tmpbuf;ú.store(tmpbuf.data(), rÖ   z!__at_align__ std::array<int32_t, z> tmpbuf_exponent;z> tmpbuf_mantissa;rÙ   rÚ   z@tmpbuf_mantissa[i] = std::frexp(tmpbuf[i], &tmpbuf_exponent[i]);z? = at::vec::Vectorized<int32_t>::loadu(tmpbuf_exponent.data(), z! = at::vec::VectorizedN<int32_t, z!>::loadu(tmpbuf_exponent.data(), rº   z ::loadu(tmpbuf_mantissa.data(), z();)r4  r£   rH   rˆ   r4   rr  Ú	tail_sizerÓ  r7   r±  rZ  r„   r[  r´  Ú_get_num_vectorsrÂ   rÄ   rE   r³  r\  r]  r^  )r·  r_  r‰   rO  r³   r`  ra  Ún_vecÚ
mantissa_trb  rR  rc  s               rX   rd  ÚCppVecOverrides.frexpñ  s_  € à˜a˜S Ð%¨°¨s°$Ð'7Ð7ˆ
ÜÑWÉJÓW×WÑWÜÑUÉ*ÓUÓUÐUä˜aŸg™gÑ&ˆÜ%&§X¡X×%7×%7Œq�x‰x×!Ò!¼Q¿X¹X×=SÑ=SˆÜ‹~ˆÜ—8‘8—<‘<×&Ñ&¬U¯[©[Ð&Ð9ˆÜ—8‘8—<‘<×&Ñ&¨Q¯W©WÐ&Ð5ˆØ×Ñ ¨!¨°bÐÑ9Ø×Ñ ¨!¨°bÐÑ9Ü—‘×)Ñ)¨!¯'©'Ó2ˆð ˜‹zð # 6 (¨!Ñ,à(¨¨°°5°'¸Ð;ð 	ð
 	�‰à˜‹zð ,¨H¨:°QÑ7à1°%°¸¸8¸*ÀAÐFô	
ð
 	�‰˜*˜ Q x j°Ð2Ô3Ø�‰�wÔØ�[‰[�]Ø�N‰NØ*¨6¨(°"´Q·X±X×5KÑ5KÐ4LÈIÐVôð �N‰N˜a˜SÐ 6´{À4Ó7HÐ6IÈÐLÔMØ�N‰NØ3´A·H±H×4JÑ4JÐ3KÐK]Ð^ôð �N‰NØ*¨6¨(°"´Q·X±X×5KÑ5KÐ4LÐL^Ð_ôð �N‰NÐ1´+¸dÓ2CÐ1DÀFÐKÔLØ—‘•Ø—‘ØVô÷ ð �N‰Nà˜A“:ð �*Ð[Ô\gÐhlÓ\mÐ[nÐnpÑqà �zÐ!BÀ5À'ÐIjÔkvÐw{Ók|Ðj}Ð}ð  Aôð
 �N‰NØ�*˜C 
˜|Ð+KÌKÐX\ÓL]ÐK^Ð^`Ðaô÷+ ð0 	�‰�uÔÜ	�‰×Ñ×Ñ Ô%ØÐ'ˆÜ"% jÖ";ÑˆIÜ�H‰H�L‰L×Ñ˜YÖ0ñ #<àÐ!Ð!÷# •ú÷ �]ús&   Ç
CN3Ê&N"Ê8A%N3Î"
N0	Î,N3Î3
Oc                 ó   ^• U4S jnU$ )Nc                  óJ  >• U(       a   e[         R                  n[        U[        5      (       d   e[	        5       nUR                  S5        U S   R                  nUR                  U5      nUR                  (       a  UR                  OUR                  n/ n[        U   nTR                  S;   n	U	(       a  SOUn
TR                  S:X  a  [        U S      OU
n
UR                  5          [        U 5       H¯  u  p¼[        U[        5      (       a„  UR                  (       d   eUR                  U:X  d   eUR                  SU SUR                   S	U S
35        UR                  U SU S[!        U5       S35        UR#                  SU S35        Mž  UR#                  U5        M±     UR                  SU
 SUR                   S35        T" U6 nUR                  S[!        U5       S35        UR                  5          UR                  SU S
35        S S S 5        S[!        U5       3nU	(       a
  SU SU S3nOUS:X  a  SU
 S3nO	SU
 SU S3nUR                  SU SU S35        S S S 5        UR                  S5        U$ ! , (       d  f       Nv= f! , (       d  f       N2= f)NrÊ  r   )r!  r$  rý  rl   r¾  éþÿÿÿr!  rœ   z> tmpbufr»   z.store(tmpbufz	.data(), rÖ   Útmpbufz[i]z> tmpbuf_out;rÙ   rÚ   ztmpbuf_out[i] = ztmpbuf_out.data(), úat::vec::VecMask<Ú,z>::fromr5   r  z>::loaduz at::vec::VectorizedN<rž  r¡  rÒ  )r4   rr  r¢   r  r7   rÂ   rˆ   r%  r$  rÓ  rH   r  rÄ   rÅ   rG   r¦   rE   r  )r›  r	  rr  r³   Ú	vec_dtyper&  rO  Úscalar_argsr‰   Úoutput_maskÚoctypeÚargidxr˜  ÚresÚ	load_argsÚload_fnr  s                   €rX   rq  Ú)CppVecOverrides._scalarize.<locals>.inner,  s—  ø€ ÞÐ�:Ü—X‘XˆFÜ˜f¤l×3Ñ3Ð3Ð3Ü“>ˆDØ�N‰N˜7Ô#Ø˜Q™Ÿ™ˆIØ×+Ñ+¨IÓ6ˆEØ'-×'7×'7�6×#Ò#¸V×=QÑ=QˆDØˆKÜ! )Ñ,ˆFØ%×.Ñ.ð 3ñ ˆKö
  +‘V°ˆFð  ×(Ñ(Ð,>Ó>ô ˜T "™XÒ&àð ð
 —‘•Ü#,¨T¦?‘K�FÜ! #¤~×6Ñ6Ø"ŸzŸzÐ)˜zØ"Ÿy™y¨IÓ5Ð5Ð5ØŸ™Ø6°v°h¸bÀ×AUÑAUÐ@VÐV^Ð_eÐ^fÐfgÐhôð Ÿ™Ø"˜e =°°¸	Ä+ÈdÓBSÐATÐTVÐWôð $×*Ñ*¨V°F°8¸3Ð+?Ö@à#×*Ñ*¨3Ö/ñ $3ð —‘Ø.¨v¨h°b¸×9MÑ9MÐ8NÈmÐ\ôñ " ;Ð/�Ø—‘Ð!5´kÀ$Ó6GÐ5HÈÐOÔPØ—[‘[•]Ø—N‘NÐ%5°c°U¸!Ð#<Ô=÷ #à1´+¸dÓ2CÐ1DÐE�	ÞØ 1°&°¸¸5¸'ÀÐI‘GØ˜a“ZØ 4°V°H¸HÐE‘Gà 6°v°h¸bÀÀÀxÐP�GØ—‘ ¨¨	°°9°+¸RÐ@ÔA÷9 ð: �N‰N˜4Ô ØˆK÷ #•]ú÷' •ús&   Ã)DJÇ=JÈAJÊ
J	ÊJÊ
J"rV   )r7  r  rq  s    ` rX   Ú
_scalarizeÚCppVecOverrides._scalarize*  s   ø€ õ4	ðl ˆrW   c                 ó  • [        [        5      n[        [        5      R                  5        HY  u  p#[	        U[
        5      (       d  M  X!;  d  M#  U R                  UR                  5      nX$l        [        X[        U5      5        M[     g r  )
r  r   rŸ  r  r¢   rü  r8  r  r  r  )r7  Úvec_varsrÇ   r  r  s        rX   Ú_initialize_scalarizeÚ%CppVecOverrides._initialize_scalarized  s^   € äœÓ(ˆÜ ¤Ó.×4Ñ4Ö6‰LˆDÜ˜&¤,×/Ó/°DÕ4HØ—~‘~ f§o¡oÓ6�Ø $”Ü˜¤<°Ó#5Ö6ò	 7rW   rV   rû  )Yr  r  r  r  r  r  rü  r¦  rÊ   rc  r&  rÂ  rÆ  rÉ  rÐ  rÓ  r×  rÛ  rÞ  rá  rå  rD  rJ  rM  rQ  rU  rY  r]  rè  r   r  r  r	  r  r  r  r'  r©  r¯  r³  r·  r¼  rÀ  rÄ  rÇ  rÖ  rÚ  rŽ  rß  râ  rç  r›  r÷  rú  r¥  rB  r*  r6  r0  r3  rj  rm  rt  rH  r>  rg  rE  r9  r-  r€  rê  rÌ  r  r  rƒ  r‡  ræ  r�  rô  r¯  rô  r¥  rš  rd  rx  r8  r<  r!  ry  rz  s   @rX   r   r   o  sV  ø† Ù8õ[ðz ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñ(ó ð(ð
 ñó ðð ñó ðð ñó ðð ñó ðð ñAó ðAð ñ	Eó ð	Eð ñ@ó ð@ð ñ@ó ð@ð ñAó ðAð ñAó ðAð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñ ó ð ð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñ0ó ð0ð ñ9ó ð9ð ñ9ó ð9ð ñFó ðFð ñAó ðAð ñó ðð ñ	!ó ð	!ð ñ#ó ð#ð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñ%ó ð%ð ñ$ó ð$ð ñ!ó ð!ð ñ!ó ð!ð ñOó ðOð ñó ðð ñó ðð ñó ðð  ñHó ðHð ñó ðð ñSó ðSð$ ñó ðð ñ1ó ð1ð ñ1ó ð1ð ñó ðð ñ	]ó ð	]ð ñó ðð óó ðð. ñ	ó ð	ð ñLó ðLð\ ñó ðð. ñ6"ó ð6"ðp ñ7ó ð7ðr ñ7ó ö7rW   r   Úcppvecc                   ó$   • \ rS rSr\S 5       rSrg)ÚCppTile2DOverridesir  c                 ó´   • [        [        R                  [        5      (       d   e[        R                  R	                  U 5      n [
        R                  X5      $ r  )r¢   r4   rr  ÚCppTile2DKernelÚtransform_indexingr   rš  )r¹  rˆ   s     rX   rš  ÚCppTile2DOverrides.index_exprs  s>   € äœ!Ÿ(™(¤O×4Ñ4Ð4Ð4Ü�x‰x×*Ñ*¨4Ó0ˆÜ×)Ñ)¨$Ó6Ð6rW   rV   N)r  r  r  r  rü  rš  r!  rV   rW   rX   r@  r@  r  s   † Øñ7ó ó7rW   r@  c                   óò  ^ • \ rS rSrSr\r\rSr	Sr
U 4S jr\\4S jrS rS5S\\   4S	 jjr\R(                  S
 5       r S6S\R.                  4S jjrS\R.                  S\4S jrS\R.                  S\R6                  4S jrS\R.                  S\R6                  4S jrS rS\R.                  S\R.                  S\S\4S jr S\S\R.                  4S jr!S5S jr"S r#S\$\%\4   S\S\S\&RN                  4S  jr(S5S\\)   4S! jjr*S" r+  S7S# jr, S8S$ jr-S% r.S& r/S' r0S( r1S) r2S* r3\4S\4S+ j5       r5S, r6\R(                  S- 5       r7S. r8S/ r9S0 r:  S9S\S1\\   S2\\R6                     4S3 jjr;S4r<U =r=$ ):Ú	CppKerneliz  a  
Base class for C++ kernel code generation in PyTorch Inductor.
This class is responsible for generating C++ code from the intermediate representation.

Args:
    args: Kernel arguments used for code generation
    num_threads: Number of threads for parallel execution
rÕ   r»   c                 óÀ  >• [         TU ]  U5        0 U l        / U l        S U l        / U l        / U l        S U l        [        5       U l	        / U l
        [        5       U l        [        5       U l        [        5       U l        [        5       U l        [        5       U l        SU l        [        5       U l        [        5       U l        ['        U R(                  U R*                  SS9U l        ['        U R(                  U R*                  SS9U l        ['        U R(                  U R*                  SS9U l        [        5       U l        [        5       U l        X l        0 U l        / U l        g )NFÚtmp_acc)Úname_prefixÚwelford_helperÚcascade_helper)r?  r@  Úactive_rangesÚinner_itervarsÚcall_rangesrd  r  Úreduction_depthr=   Úreduction_prefixÚreduction_prefix_generatorsÚreduction_suffixÚparallel_reduction_prefixÚparallel_reduction_suffixÚlocal_reduction_initÚlocal_reduction_storesÚis_reductionÚnon_parallel_reduction_prefixÚnon_parallel_reduction_suffixr8   Únewvar_prefixÚsuffixÚreduction_cseÚwelford_helper_cseÚcascade_helper_cseÚpreloadsÚ
poststoresÚnum_threadsÚreduction_omp_decÚreduction_var_names)rA  r›  ra  rD  s      €rX   r@  ÚCppKernel.__init__‰  s5  ø€ Ü‰Ñ˜Ôð HJˆÔð 35ˆÔØ=AˆÔØ(*ˆŒØ,.ˆŒØ#ˆÔÜ .Ó 0ˆÔð <>ˆÔ(Ü .Ó 0ˆÔÜ)7Ó)9ˆÔ&Ü)7Ó)9ˆÔ&Ü$2Ó$4ˆÔ!Ü&4Ó&6ˆÔ#Ø!ˆÔÜ-;Ó-=ˆÔ*Ü-;Ó-=ˆÔ*Ü  ×!3Ñ!3°T·[±[ÈiÑXˆÔÜ"%Ø×Ñ §¡Ð9Iñ#
ˆÔô #&Ø×Ñ §¡Ð9Iñ#
ˆÔô 'Ó(ˆŒÜ(Ó*ˆŒØ&ÔØ=?ˆÔØ.0ˆÕ rW   c                 óJ  • [         R                  R                  (       a,  U R                  (       d  U R                  R	                  S5        U S3n[         R                  R                  (       a  SO	[        5       nU S3n	U R                  R	                  U SU SU" X45       S35        U R                  R                  [        UUUUUU5      5        U R                  R	                  U	 SU S35        U R                  R                  SU S	3S
SU SU" X1X”S9 S3S/5        g )Nz(int max_threads = omp_get_max_threads();Ú_localÚmax_threadsz	_arr[tid]r˜   rº   r»   zfor (int tid = 0; tid < z; tid++)rÛ   rÜ   ©rª   r€   )r   rð  Údynamic_threadsrS  rÂ   r+   rU  r\  râ   rV  rT  rÞ   )
rA  r²   rÒ   r‡   rˆ   Úreduction_combine_fnÚreduction_init_fnÚ	acc_localra  Úacc_local_in_arrays
             rX   Ú_gen_parallel_reduction_buffersÚ)CppKernel._gen_parallel_reduction_buffers²  s6  € ô �:‰:×%×%¨d×.L×.LØ×*Ñ*×4Ñ4Ø:ôð �e˜6�Nˆ	ä#ŸZ™Z×7×7‰MÔ=QÓ=Sð 	ð !$˜u IÐ.ÐØ×!Ñ!×+Ñ+Øˆj˜˜)˜ CÑ(9¸.Ó(PÐ'QÐQRÐSô	
ð 	×&Ñ&×-Ñ-Ü"ØØØØØØ!óô		
ð 	×#Ñ#×-Ñ-Ð1CÐ0DÀCÈ	À{ÐRSÐ.TÔUØ×&Ñ&×1Ñ1à*¨;¨-°xÐ@ØØ�s�e˜3Ñ3°NÐI[ÑmÐnÐnoÐpØð	õ	
rW   c                 ó\   • U R                    H  n[        U R                  X S35        M     g )Nrf  )rc  rç   Ústores)rA  Úvar_names     rX   Ú%update_stores_with_parallel_reductionÚ/CppKernel.update_stores_with_parallel_reductionÛ  s'   € Ø×0Ô0ˆHÜ˜TŸ[™[¨(°jÀÐ4GÖHò 1rW   r³   c                 ó¾  • Ub   e[        5       n[        R                  " 5        n[        U S5      (       aK  UR	                  U R
                  5        U R                  U5        UR                  UR                  5       5        UR	                  U R                  5        UR	                  U R                  5        UR	                  U R                  5        S S S 5        [        U S5      (       a  UR	                  U R                  5        U R                  (       a4  U R                   H$  nU R                  U   u  pE[        XU S3XE5      nM&     U$ ! , (       d  f       N�= f)NÚcodegen_inner_loopsÚ_tail)r7   rÀ   rÁ   r¥   r\  r_  rv  rÃ   rÄ   Úloadsr³  rq  r`  rM  rL  rÐ   )rA  r³   rÌ   rg  ÚstartÚends         rX   Úgen_bodyÚCppKernel.gen_bodyß  s  € Ø‰|Ðˆ|Ü‹~ˆÜ×!Ò!Ô# uÜ�tÐ2×3Ñ3Ø—‘˜DŸM™MÔ*Ø×(Ñ(¨Ô.Ø×#Ñ# D§K¡K£MÔ2Ø�K‰K˜Ÿ
™
Ô#Ø�K‰K˜Ÿ™Ô%Ø�K‰K˜Ÿ™Ô$÷ $ô �4Ð.×/Ñ/Ø�K‰K˜Ÿ™Ô(à××Ø×*Ô*�Ø!×/Ñ/°Ñ4‘
�Ü1°$À¸uÀE¸]ÈEÓW’ñ +ð ˆ÷ $Õ#ús   ¥B.EÅ
Ec              #   ó0  #   • U R                   nU(       ac  [        R                  " X5      n[        U[        5      (       a8  UR
                  n[        U[        5      (       d   e[        R                  Ul	        Xl          Uv •  X l         g! X l         f = f7f)z>Context manager to add an additional mask to loads and stores.N)
Ú
_load_maskr2   r]  r¢   r3   rï   rG   r„   rl   rˆ   )rA  r   Úpriors      rX   r¥  ÚCppKernel.maskedó  sp   é € ð —‘ˆÞÜ—8’8˜DÓ(ˆDÜ˜$¤×)Ñ)Ø—z‘z�Ü! $¬×7Ñ7Ð7Ð7ô #ŸZ™Z�”
àŒð	$ØŠJà#�Oø˜e�Oüs   ‚A=BÂ B ÂBÂBÂBr’   c                 óL   • U R                   U   nXUU-  U-   0n[        X5      nU$ r  )r  r0   )rA  r’   ÚscaleÚitervar_idxrP  r§   rù   rú   s           rX   Úscale_index_with_offsetÚ!CppKernel.scale_index_with_offset  s3   € ð �m‰m˜KÑ(ˆØ %™K¨&Ñ0Ð1ˆÜ˜uÓ2ˆ	ØÐrW   r¸   c                 ó6   • [        U R                  U5      5      $ )zŽ
Convert an index expr to a string that can be used in cpp code.
e.g. a sympy expression "s2" may actually appear as "ks1" in the cpp kernel.
)rD   r˜  ©rA  r’   s     rX   Úindex_to_strÚCppKernel.index_to_str  s   € ô
 �T×)Ñ)¨%Ó0Ó1Ð1rW   Úitervarc                 óF   ^ ^• [        UU 4S jUR                   5       5      $ )zM
Check if an index has free symbol CppCSEVariable that depends on `itervar`.
c              3   óD  >#   • U  H•  nUR                   TR                  R                  ;   d  M)  [        TR                  R                  UR                      [        5      (       d  Ma  TR                  R                  UR                      R                  T5      v •  M—     g 7fr  )rÇ   r±  Úvarname_mapr¢   rG   Ú
depends_on)r*  ÚsrŠ  rA  s     €€rX   r,  Ú6CppKernel.index_indirect_depends_on.<locals>.<genexpr>  su   øé € ð 
â'�Ø�v‰v˜Ÿ™×-Ñ-Ñ-ó =ô ˜4Ÿ8™8×/Ñ/°·±Ñ7¼×Hó =ˆD�H‰H× Ñ  §¡Ñ(×3Ñ3°G×<Ð<Ú'ùs   ƒ(B ¯4B Á'9B )rg   Úfree_symbols©rA  r’   rŠ  s   ` `rX   Úindex_indirect_depends_onÚ#CppKernel.index_indirect_depends_on  s%   ù€ ô õ 
à×'Ò'ó
ó 
ð 	
rW   c                 óN   • X!R                   ;   =(       d    U R                  X5      $ r  )r‘  r“  r’  s      rX   Úindex_depends_onÚCppKernel.index_depends_on   s(   € Ø×,Ñ,Ñ,÷ 
°×0NÑ0NØó1
ð 	
rW   c                 óT   • [        [        U R                  U R                  5      5      $ r  )rþ  r]  r  rd  rG  s    rX   Ú
var_rangesÚCppKernel.var_ranges%  s   € Ü”C˜Ÿ™ t§{¡{Ó3Ó4Ð4rW   r¹  rO  ÚlowerÚupperc                 ó,  • U(       d  U(       d  g [        U[        R                  5      nU(       aJ  [        R                  " U[
        R                  5      R                  n[        R                  R                  nOŠ[        R                  R                  n U R                  [        R                  l
        [        R                  " U[
        R                  5      R                  nU[        R                  l
        U R                  nU(       a.  [        R                  R                  U R                  U5      5      OS n	U R                  Xc(       a  SOS X�R                  5      n
U R                   R#                  XzSS9  g ! U[        R                  l
        f = f)NÚ0F)Ú
assignment)r   r   ÚTMPr2   rš  r„   r  rï   r4   rr  r³  rx  Úsexprr˜  Úindirect_assertr~  r±  r²  )rA  r¹  rO  r›  rœ  Úindirectrº  rã   Úprior_computeÚsize_strrÈ   s              rX   Úcheck_boundsÚCppKernel.check_bounds(  s  € ö žØä& t¬T¯X©XÓ6ˆÞä—^’^ D¬%¯+©+Ó6×<Ñ<ˆFÜ—X‘X×%Ñ%‰Fô ŸH™H×,Ñ,ˆMð1Ø#'§:¡:”—‘Ô ÜŸš¨¬e¯k©kÓ:×@Ñ@�à#0”—‘Ô Ø—Z‘ZˆFæAF”1—8‘8—>‘> $×"6Ñ"6°tÓ"<Ô=ÈDˆà×#Ñ#Ø�5‘C d¨H·o±oó
ˆð 	�‰×Ñ˜&°5ÐÒ9øð $1”—‘Õ ús   ÂAE< Å<FrÇ   c                 ó.  • U R                   R                  U5      nU R                  U5      nU S[        U5       S3nU R                  R                  U R                  U[        R                  R                  U5      S9nUR                  SXU40 5        U$ )NÚ[Ú]r  r�  )r›  Úinputr˜  rE   r±  r²  rx  r4   ÚgraphÚ	get_dtyper´  )rA  rÇ   r’   r§   rÈ   rº  s         rX   r�  ÚCppKernel.loadH  s†   € Ø�i‰i�o‰o˜dÓ#ˆØ×$Ñ$ UÓ+ˆØ��aœ EÓ*Ð+¨1Ð-ˆØ—‘×"Ñ" 4§:¡:¨t¼1¿7¹7×;LÑ;LÈTÓ;RÐ"ÐSˆØ×Ñ˜f t°5Ð&9¸2Ô>ØˆrW   c                 ó  • SU;   d   eU R                   R                  U5      nU R                  U5      nUc  U S[        U5       SU S3nO�US:X  a‰  [        R
                  R                  (       d%  U R                  S:X  a  U S[        U5       SU S3nOS[        R                  R                  U5      nS[        U    S	U S
3nSU S[        U5       SU S3nO[        SU 35      eU R                  R                  [        X5      5        g )NÚbufr©  ú] = r»   Ú
atomic_addr5   z] += zstatic_cast<r½  r–   zatomic_add(&z], rÖ   ústore mode=)r›  Úoutputr˜  rE   r   rð  ri  ra  r4   r¬  r­  rH   rø  rq  rÂ   r;   )rA  rÇ   r’   rï   Úmoder§   rÈ   rˆ   s           rX   ÚstoreÚCppKernel.storeP  s  € Ø˜‹}Ðˆ}Ø�i‰i×Ñ˜tÓ$ˆØ×$Ñ$ UÓ+ˆØ‰<Ø�U˜!œK¨Ó.Ð/¨t°E°7¸!Ð<‰DØ�\Ó!Ü—:‘:×-×-°$×2BÑ2BÀaÓ2GØ˜˜a¤¨EÓ 2Ð3°5¸¸¸qÐA‘äŸ™×)Ñ)¨$Ó/�à&¤|°EÑ':Ð&;¸2¸e¸WÀAÐF�Ø% c U¨!¬K¸Ó,>Ð+?¸sÀ5À'ÈÐL‘ä%¨°D°6Ð&:Ó;Ð;Ø�‰×Ñœl¨4Ó6Õ7rW   c                 óH   • U R                   R                  SU SU S35        g )Nr¡  z" ? 0 : (throw std::runtime_error("z"), 0));)r³  rÂ   )rA  ÚcondÚmsgs      rX   Údevice_assert_asyncÚCppKernel.device_assert_asyncb  s'   € Ø�‰×ÑØ�ˆvÐ7¸°u¸HÐEõ	
rW   r²   rÒ   Úrtyperˆ   c                 óF   ^^^^^• SS[         [           4UUUUU4S jjjnU$ )NrO  c                 óN   >• U c  T ST ST" TT5       S3$ [        TTTTU T5      $ )Nr˜   rº   r»   )râ   )rO  r²   rÒ   rˆ   rß   r½  s    €€€€€rX   rq  Ú.CppKernel._gen_reduction_prefix.<locals>.inneru  sH   ø€ Ø‰|Ø"˜ 1 S E¨©W°U¸EÓ-BÐ,CÀ1ÐEÐEä-ØØØØØØóð rW   r  )r   rn   )rA  r²   rÒ   r½  rˆ   rß   rq  s    ````` rX   Ú_gen_reduction_prefixÚCppKernel._gen_reduction_prefixg  s   ü€ ñ	œ¤™÷ 	ô 	ð ˆrW   c                 ól   • U R                    H$  nU R                  R                  U" U5      5        M&     g r  )rQ  rP  r\  )rA  rO  Úgen_fns      rX   Úfinalize_reduction_prefixÚ#CppKernel.finalize_reduction_prefix„  s*   € Ø×6Ô6ˆFØ×!Ñ!×(Ñ(©°«Ö6ò 7rW   c                 ó  • US:X  a  gUS:X  aã  U[         R                  :X  aÏ  U R                  c   e[        R                  " [
        R                  U R                  U R                  S  5      nSn [        R                  R                  R                  U5      nXe:”  a*  [        R                  R                  R                  XT5        g[        R                  R                  R                  XE5        g! [         a     gf = f)Nrh   Trd   é   F)r„   ro   rN  r`  ra  rb  rc  rO  r4   r¬  ÚsizevarsÚ	size_hintÚ	ExceptionÚcheck_ltÚ	check_leq)rA  r‡   rˆ   Ú
use_scalarÚreduction_sizeÚ
chunk_sizeÚreduction_size_hints          rX   Úneed_use_acc_helperÚCppKernel.need_use_acc_helperˆ  sÝ   € ð Ð-Ó-Øð ˜UÓ" u´·±Ó';Ø×#Ñ#Ñ/Ð/Ð/Ü&×-Ò-Ü—‘˜d×.Ñ.¨t×/CÑ/CÐ/EÐFóˆNð
 ˆJðÜ&'§g¡g×&6Ñ&6×&@Ñ&@ÀÓ&PÐ#ð #Ó/ä—‘× Ñ ×)Ñ)¨*ÔEØä—‘× Ñ ×*Ñ*¨>ÔFØøô ó Ùðús   Á0)C2 Ã2
C?Ã>C?c           
      ó€  • U(       a  [        X55      OUn[        U5      nUS;   d   eSn	[        Xy5      n
US:X  a  SOSnU(       a
  [        U   nO+[        U S5      (       a  U R	                  U5      O[        U   nU SU SU	 S	U S
U S3
nUS:X  a  U$ [        U
[        R                  5      (       a  U
S::  a  SU 3$ U$ )N)rh   rd   rÈ  rh   ÚWelfordHelperÚCascadeSumHelperr  Ú<rœ   r   r¡  rÖ   rd   r5   zstatic )r   rE   rH   r¥   r  r¢   rõ   re  )rA  r‡   r©   Úhelper_rangerˆ   ra  rÎ  Únum_range_threadÚnum_range_thread_exprrÐ  Ú
num_chunksÚhelper_typeÚh_typeÚhelper_init_lines                 rX   Ú_acc_helper_initÚCppKernel._acc_helper_init©  s  € ö 3>ŒG�LÔ.À<ð 	ô !,Ð,<Ó =ÐØÐ!:Ó:Ð:Ð:Øˆ
ÜÐ-Ó:ˆ
ð Ð!1Ó1ñ à#ð 	ö
 Ü! %Ñ(‰Fô ˜4 ×1Ñ1ð ×"Ñ" 5Ô)ä! %Ñ(ð ð ˆm˜1˜V˜H B z l°"°Z°LØØ$Ð%Øðð 	ð ˜UÓ"Ø#Ð#Ü�j¤%§-¡-×0Ñ0°ZÀ1³_ð Ð-Ð.Ð/Ð/à#Ð#rW   c           
      óJ  • [         R                  R                  (       a  SO	[        5       nU R                  R                  U R                  XXES U5      5        U R                  R                  U R                  XXEXv5      5        U(       a  UOU S3nUS:X  aI  U R                  R                  U SU SU S35        U R                  R                  U SU SU S35        g U R                  R                  U S	U S35        U R                  R                  U S
U S35        g )Nrg  Ú_vecrh   z = welford_combine(r•   rÖ   z_local = welford_combine(z	_local, &z = cascade_sum_final(&z_local = cascade_sum_final(&)
r   rð  ri  r+   rX  rÂ   rß  rU  rY  rV  )	rA  r‡   r²   r©   rØ  rˆ   rÎ  ra  r  s	            rX   Ú_use_acc_helperÚCppKernel._use_acc_helperÕ  s7  € ô $ŸZ™Z×7×7‰MÔ=QÓ=Sð 	ð 	×*Ñ*×4Ñ4Ø×!Ñ!Ø¨LÀÀzóô	
ð
 	×!Ñ!×+Ñ+Ø×!Ñ!Ø¨LÀóô	
ö
 #‘¨3¨%¨t¨ˆØÐ-Ó-Ø×.Ñ.×8Ñ8Ø�(Ð-¨f¨X°S¸¸ÀBÐGôð ×'Ñ'×1Ñ1Ø�(Ð3°F°8¸9ÀZÀLÐPRÐSõð ×.Ñ.×8Ñ8Ø�(Ð0°°¸BÐ?ôð ×'Ñ'×1Ñ1Ø�(Ð6°z°lÀ"ÐEõrW   c                 ó0  • US;   nX#U4nX`R                   R                  ;   a  U R                   R                  U   $ U R                   R                  U R                  SU 3SS9nU R                  R                  U 5        SU l        U(       a  UOUn[        X85      n	U R                  R                  U R                  XyX8[        5      5        U R                  X1S5      (       aØ  [        R                  " [        R                  U R                   U R"                  S  5      n
US:X  a)  U R$                  R                  U R&                  SU 3SS9nO(U R(                  R                  U R&                  SU 3SS9nSU 3nU R+                  UUUU
USS9  U R,                  R/                  U S	[1        X7XL5       S
35        O¥U R"                  c   eU R2                  U R"                     n[5        U R"                  S-   [7        U R2                  5      5       H$  nXÐR                   U   -  U R2                  U   -   nM&     U R,                  R/                  U S	[1        X7XMS9 S
35        U R9                  XyX85        [;        X75      nXðR                   R                  U'   U$ )N©rb   ra   ú
reduction F©ÚwriteTrh   Úscalar_©rÎ  rº   r»   r5   )r’   )r\  Úreduction_cacher²  rx  rc  r  rW  r‘   rQ  rÁ  r�   rÒ  r`  ra  rb  rc  rd  rO  r]  r³  r^  rã  rq  rÂ   r°   r  r^  rÓ   rn  r¤   )rA  rˆ   rª   r‡   rï   Úargmax_or_argminÚreduction_keyr²   Ú
init_dtyperÒ   rÏ  r©   Úscalar_helper_valr’   ræ   r  s                   rX   Ú	reductionÚCppKernel.reductionõ  s   € Ø)Ð-AÑAÐØ!°5Ð8ˆØ×.Ñ.×>Ñ>Ó>Ø×%Ñ%×5Ñ5°mÑDÐDà× Ñ ×)Ñ)Ø�J‰J˜* ] OÐ4¸Eð *ð 
ˆð 	× Ñ ×'Ñ'¨3¨%Ô1Ø ˆÔÞ"2‘Y¸ˆ
Ü% nÓAˆØ×(Ñ(×/Ñ/Ø×&Ñ&Ø˜~¼>óô	
ð ×#Ñ# N¸4×@Ñ@ä&×-Ò-Ü—‘˜dŸk™k¨$×*>Ñ*>Ð*@ÐAóˆNð Ð!1Ó1Ø!×4Ñ4×=Ñ=Ø—L‘L J¨}¨oÐ">Àeð >ð ‘
ð "×4Ñ4×=Ñ=Ø—L‘L J¨}¨oÐ">Àeð >ð �
ð #*¨*¨Ð 6ÐØ× Ñ ØØØ!ØØØð !ñ ð �K‰K×!Ñ!Ø�%�sÔ,¨^À%Ó[Ð\Ð\]Ð^õð ×'Ñ'Ñ3Ð3Ð3Ø—M‘M $×"6Ñ"6Ñ7ˆEÜ˜4×/Ñ/°!Ñ3´S¸¿¹Ó5GÖH�Ø§¡¨A¡Ñ.°·±¸qÑ1AÑA’ñ Ià�K‰K×!Ñ!Ø�%�sÔ,¨^À%ÑUÐVÐVWÐXôð 	×,Ñ,¨S¸NÔWÜ" >Ó7ˆØ<B×Ñ×*Ñ*¨=Ñ9ØˆrW   c                 óÈ   • U R                  U5      nU R                  R                  U5      nU R                  R	                  [        X S[        U5       SU S35      5        g )Nr©  r±  r»   )r˜  r›  r´  rR  rÂ   r;   rE   )rA  rÇ   r’   rï   r§   s        rX   Ústore_reductionÚCppKernel.store_reduction1	  s[   € Ø×$Ñ$ UÓ+ˆØ�i‰i×Ñ˜tÓ$ˆØ×Ñ×'Ñ'Ü˜  a¬°EÓ(:Ð';¸4À¸wÀaÐHÓIõ	
rW   c                 óÄ  • U R                   (       al  U R                   [        U5      [        U5      -   :X  d+   U R                    S[        U5       S[        U5       35       eU R                  [        U5      :X  d   eO¨[        U5      [        U5      -   U l         U R                    Vs/ s H  o0R	                  U5      PM     snU l        [        [        U R
                  5      5       Vs/ s H  n[        [        R                  U5      PM     snU l
        [        U5      U l        U R                  S U R                   U R                  U R                  S  4$ s  snf s  snf )NrA  ré   )rN  r£   rO  rÓ   r˜  rd  r^  r.   r   ÚXBLOCKr  )rA  ÚlengthsÚreduction_lengthsr·  rp  s        rX   Ú
set_rangesÚCppKernel.set_ranges8	  s;  € Ø××Ø×#Ñ#¤u¨W£~¼Ð>OÓ8PÑ'PÓPð Ø×#Ñ#Ð$ D¬¨w«Ð(8¸¼EÐBSÓ<TÐ;UÐVóÐPð ×'Ñ'¬3¨w«<Ó7Ð7Ñ7ä$ W›~´Ð6GÓ0HÑHˆDÔØ<@×<LÒ<LÓMÒ<L°q×/Ñ/°Ö2Ñ<LÑMˆDŒKô œs 4§;¡;Ó/Ô0óâ0�Aô /¬t¯{©{¸AÖ>Ù0ñˆDŒMô
 $' w£<ˆDÔ à�M‰MÐ0˜D×0Ñ0Ð1Ø�M‰M˜$×.Ñ.Ð0Ð1ð
ð 	
ùò Nùòs   Â(EÃ*$Ec                 ó–   • U R                   c   e[        R                  R                  R	                  [        U R                   5      SS9$ )Né    ©Úfallback)rN  r4   r¬  rÉ  rÊ  r/   rG  s    rX   rÊ  ÚCppKernel.size_hintL	  sF   € Ø×ÑÑ+Ð+Ð+Ü�w‰w×Ñ×)Ñ)Ü˜$×*Ñ*Ó+°dð *ð 
ð 	
rW   c                 óê  ^^^^^^^^^^• [        U [        5      (       d   e[        5       mU R                  c   e[        UR                  [
        5      (       a+  UR                  R                  UR                  5       T5      mO U R                  UR                  5       T5      mUR                  S L=(       a#    UR                  TR                     R                  m[        R                  " 5        nTR                  (       a;  T(       a  TR                  5         OTR                  T5        UR!                  T5        O:TS:”  a4  TR#                  5       (       a  UR%                  TR'                  5       5        S[(        4UUU4S jjmSS jm SS[(        S[*        4UUUUUU4S jjjmSS[(        S[*        4UU4S jjjm  SS[(        S[*        S[,        4UU4S	 jjjmUR%                  TR'                  5       5        [        UR                  [
        5      (       Ga/  [        [.        R0                  [2        5      (       Ga  [.        R0                  R4                  (       aì  [.        R0                  R4                  nUR7                  5        H¾  n[9        UR;                  5       R<                   Vs/ s H  nU R?                  U5      PM     sn5      n[@        UR;                  5       RB                     n	S
U	 S[E        U5       S3n
URG                  5       nTRI                  SU	 SU SU
 S35        TRI                  U	 SU SU S35        MÀ     T" U5        S S S 5        g s  snf ! , (       d  f       g = f)Nr5   Ú
_loop_nestc                 óÖ  >^ • U U4S jnT R                  5       n[        U[        5      (       a  UR                   H  m T" T 5        M     g [        U[        5      (       d   eT R
                  b  U" 5       (       a  UR                  5         [        R                  " 5        nUR                  TR                  5       5        UR                  T5        S S S 5        g ! , (       d  f       g = f)Nc                  óš   >• TR                   (       d   eTR                   TR                     n U R                  =(       a    U R                  $ r  )rX  r  rW  Úparallel)Úrootr  Ú	par_depths    €€rX   Úis_parallel_reductionÚOCppKernel.codegen_loops_impl.<locals>.gen_kernel.<locals>.is_parallel_reductionp	  s=   ø€ Ø%×+×+Ð+Ð+Ø%×+Ñ+¨I×,AÑ,AÑB�DØ×,Ñ,×>°·±Ð>rW   )Ú
get_kernelr¢   ro  rq  ÚCppKernelProxyrX  rs  rÀ   rÁ   rÃ   rÄ   r{  )r  r  rr  rÌ   r³   Úgen_loop_nestr  s   `   €€€rX   Ú
gen_kernelÚ0CppKernel.codegen_loops_impl.<locals>.gen_kernelo	  s¬   ù€ ö?ð
 $×.Ñ.Ó0�Ü˜fÔ&:×;Ñ;Ø&,§l¤l˜
Ù% jÖ1ò '3ô & f¬n×=Ñ=Ð=Ð=Ø!×'Ñ'Ñ3Ñ8M×8OÑ8OØ×DÑDÔFÜ#×-Ò-Ô/°5Ø×+Ñ+¨D¯K©K«MÔ:ØŸ™¨Ô-÷ 0×/Ö/ús   Â 1CÃ
C(c                 óà   • U(       a5  U R                   nU(       a  U R                  U-   nU$ U R                  U-   nU$ U R                  nU(       a  X@R                  -   nU$ X@R
                  -   nU$ r  )rR  rT  rY  rP  rS  rX  )rr  r  Ú	is_suffixr[  Úprefixs        rX   Úget_reduction_prefix_suffixÚACppKernel.codegen_loops_impl.<locals>.get_reduction_prefix_suffix�	  sy   € ÞØ#×4Ñ4�FÞØ!'×!AÑ!AÀFÑ!J˜ð "�Mð "(×!EÑ!EÈÑ!N˜Ø!�Mà#×4Ñ4�FÞØ!'×*JÑ*JÑ!J˜ð "�Mð "(×*NÑ*NÑ!N˜Ø!�MrW   Údepthc           	      ó€  >• U R                  5       nU R                  (       d   eU R                  U   n[        R                  " 5        nUR                  (       aO  U(       dH  T	" X4R
                  SS9nU(       a  UR                  TR                  5       5        TR                  U5        T
(       aa  UR
                  (       aP  TR                  T5        UR                  (       a.  UR                  (       d   eTR                  UR                  5        T" X5        T
(       aM  UR
                  (       a<  UR                  (       a  TR                  UR                  5        TR                  5         UR                  (       a'  U(       d   TR                  T	" X4R
                  SS95        S S S 5        g ! , (       d  f       g = f)NF)r  T)r
  rX  rÀ   rÁ   rW  r  rÃ   rÄ   r\  rU  rV  Úclose)r  r  Úin_reductionrr  ÚloopÚstack_outerrP  r³   Úgen_loop_atr  Úis_reduction_loopÚthreadsÚworksharings          €€€€€€rX   Úgen_loop_with_reductionÚ=CppKernel.codegen_loops_impl.<locals>.gen_loop_with_reduction‘	  s2  ø€ ð $×.Ñ.Ó0�Ø!×'×'Ð'Ð'Ø!×'Ñ'¨Ñ.�Ü×)Ò)Ô+¨{Ø×(×(¶Ù+FØ"§M¡M¸Uñ,Ð(ö ,Ø'×5Ñ5°d·k±k³mÔDØŸ™Ð$4Ô5Þ(¨T¯]¯]Ø#×,Ñ,¨WÔ5Ø!×6×6Ø#)×#@×#@Ð@Ð#@Ø ŸK™K¨×(CÑ(CÔDá 
Ô2æ(¨T¯]¯]Ø!×8×8Ø ŸK™K¨×(EÑ(EÔFØ#×)Ñ)Ô+Ø×(×(¶ØŸ™Ù7Ø &¯©Àñô÷+ ,×+Ö+ús   Á	EF/Æ/
F=c                 ón  >• [         R                  " 5        nU R                  (       d   eU R                  U   nUR                  5       nUc
   S S S 5        g TR	                  U5        UR                  TR                  5       5        T" XS-   UR                  5        S S S 5        g ! , (       d  f       g = fró   )rÀ   rÁ   rX  ÚlinesrÞ   rÃ   rÄ   rW  )r  r  rÌ   r  Ú
loop_linesr³   r  s        €€rX   r  Ú1CppKernel.codegen_loops_impl.<locals>.gen_loop_at²	  sŽ   ø€ Ü×)Ò)Ô+¨uØ%×+×+Ð+Ð+Ø%×+Ñ+¨EÑ2�DØ!%§¡£�JØ!Ñ)Ø÷ ,Ð+ð —O‘O JÔ/Ø×'Ñ'¨¯©«Ô6Ù! *°a©i¸×9JÑ9JÔK÷ ,×+Ö+ús   —7B&ÁAB&Â&
B4r  c                 óv   >• U R                   b  U[        U R                   5      :X  a	  T" U 5        g T" XU5        g r  )rX  rÓ   )r  r  r  r  r  s      €€rX   r  Ú3CppKernel.codegen_loops_impl.<locals>.gen_loop_nest½	  s5   ø€ ð
 ×#Ñ#Ñ+¨u¼¸J×<LÑ<LÓ8MÓ/MÙ˜zÕ*á+¨J¸|ÕLrW   zstd::make_unique<z []>(r–   zstd::unique_ptr<z	 []> buf_rº   r»   z* z = buf_z.get();)FF)r   F)r   )%r¢   r  r+   rN  rr  ro  Údecide_parallel_depthÚmax_parallel_depthrX  r  rW  rÀ   rÁ   r  r  r  Úmark_parallelÚsinglerÃ   rÄ   r]  rn   rl   r4   Úlocal_buffer_contextrK   Úlocal_buffersÚvaluesr/   Ú
get_layoutrO  r˜  rH   rˆ   rD   Úget_namer\  )rA  r_  r³   r  rÌ   r+  Úlocal_bufferÚsize_valÚlocal_buf_sizeÚlocal_buf_dtypeÚallocateÚlocal_buffer_namer  r  r  r  r  r  r  r  s     ``        @@@@@@@@rX   Úcodegen_loops_implÚCppKernel.codegen_loops_implR	  s!  ÿù€ Ü˜$¤×/Ñ/Ð/Ð/Ü&Ó(ˆØ×ÑÑ+Ð+Ð+Ü�i×&Ñ&Ô(<×=Ñ=Ø!×(Ñ(×>Ñ>Ø×,Ñ,Ó.°ó‰Ið ×2Ñ2Ø×,Ñ,Ó.°óˆIð
 �O‰O 4Ð'÷ DØ—‘ 	× 5Ñ 5Ñ6×CÑCð 	ô ×!Ò!Ô# uØ×'×'Þ$à×%Ñ%Õ'à×(Ñ(¨Ô1Ø×'Ñ'¨	Õ2Ø˜1“Ø×%Ñ%×'Ñ'Ø×'Ñ'¨¯©«Ô6ð.¤x÷ .ñ .ô$"ð" DIñÜ$ðÜ-0÷õ ñB	L¬ð 	L¼÷ 	Lñ 	Lð Ø%*ñMÜ$ðMäðMô #÷Mñ Mð ×Ñ §¡£Ô.ô ˜9×+Ñ+Ô-A×BÒBÜœq×5Ñ5Ô7I×JÒJÜ×*Ñ*×8×8ô !"× 6Ñ 6× DÑ D�Ø$1×$8Ñ$8Ö$:�Lä%2ð -9×,CÑ,CÓ,E×,JÒ,Jóâ,J ð !×0Ñ0°Ö:Ù,Jñó&�Nô '3°<×3JÑ3JÓ3L×3RÑ3RÑ&S�OØ!2°?Ð2CÀ5ÌÈ~ÓI^ÐH_Ð_`Ða�HØ(4×(=Ñ(=Ó(?Ð%Ø—K‘KØ*¨?Ð*;¸9ÐEVÐDWÐWZÐ[cÐZdÐdeÐfôð —K‘KØ*Ð+¨2Ð.?Ð-@ÀÐHYÐGZÐZaÐböñ %;ñ" ˜)Ô$÷} $Ð#ùò`÷a $Õ#ús    Ã2F>M$Ê0MËBM$ÍM$Í$
M2c                 óR   • [         R                  U 5      nU R                  X1U5        g r  )r]  Úbuildr5  )rA  r³   r  r_  s       rX   Úcodegen_loopsÚCppKernel.codegen_loopsã	  s    € Ü—N‘N 4Ó(ˆ	Ø×Ñ 	°Õ=rW   c                 óD   • [         R                  R                  (       a  gg)NÚAOTI_TORCH_CHECKÚTORCH_CHECK)r4   r¬  Úaot_moderG  s    rX   Úassert_functionÚCppKernel.assert_functionç	  s   € ä�7‰7××Ø%à rW   c                 ó  • U R                   c   eU R                   UR                  UR                  UR                  -    nU R                  5       nSnSnU Hj  n[        R
                  R                  R                  USS9nUSU-  :¼  d  XR:X  a    O3XB-  [        R                  R                  :  a    OUS-  nXX-  nXH-  nMl     [        R                  R                  (       a  US:X  a  [        U5      S:”  a  Sn[        XaR                  S9$ )Nr5   r   rý  rþ  r   ©r  r  )rN  r  r  rÊ  r4   r¬  rÉ  Úoptimization_hintr   rð  Úmin_chunk_sizeri  rÓ   r  )	rA  r'  r  rd  ÚseqÚparr  r¹  Úhints	            rX   r&  ÚCppKernel.decide_parallel_depthî	  sþ   € Ø×ÑÑ+Ð+Ð+Ø×!Ñ!Ø×*Ñ*Ø"×.Ñ.Ð1C×1RÑ1RÑRð
ˆð
 �n‰nÓˆØˆØˆÛˆDÜ—7‘7×#Ñ#×5Ñ5°dÀTÐ5ÐJˆDØ�a˜'‘kÓ! S£^ÙØ‰~¤§
¡
× 9Ñ 9Ó9áØ�Q‰JˆEØ‰KˆCØ‰KŠCñ ô �:‰:×%×%¨%°1«*¼¸V»Àq»ØˆEÜØ ×.LÑ.Lñ
ð 	
rW   c              #   ó  #   • U R                   U R                  U R                  U R                  4n[	        5       U l         [	        5       U l        [	        5       U l        U R                  R                  5       U l        S v •  U R                  R                  U R                   5        U R                  R                  U R                  5        U R                  R                  U R                  5        Uu  U l         U l        U l        U l        g 7fr  )rx  r³  rq  r±  r=   ÚclonerR  r\  )rA  r  s     rX   Úwrite_to_suffixÚCppKernel.write_to_suffix
  s´   é € à—‘˜TŸ\™\¨4¯;©;¸¿¹ÐAˆÜ#Ó%ˆŒ
Ü%Ó'ˆŒÜ$Ó&ˆŒØ—8‘8—>‘>Ó#ˆŒÛØ×Ñ×$Ñ$ T§Z¡ZÔ0Ø×Ñ×$Ñ$ T§\¡\Ô2Ø×Ñ×$Ñ$ T§[¡[Ô1Ø<AÑ9ˆŒ�T”\ 4¤;°µùs   ‚D
Dc                 ó   • [        U0 UD6$ r  )rG   )rA  r›  r	  s      rX   Úcreate_cse_varÚCppKernel.create_cse_var
  s   € Ü˜tÐ. vÑ.Ð.rW   c                 ó"   • S[         U    SU S3$ )Nzc10::convert<r½  r–   )rH   )rA  Úsrcrˆ   rª   s       rX   r°  ÚCppKernel.get_to_dtype_expr
  s   € Øœ|¨EÑ2Ð3°2°c°U¸!Ð<Ð<rW   c                 ó^   • U R                  X2U5      nU R                  R                  XQ5        g r  )r°  r±  r^  )rA  ÚdstÚ	dst_dtyperQ  rª   r¹  s         rX   rµ  ÚCppKernel.cache_dtype_convert
  s$   € Ø×%Ñ% c°iÓ@ˆØ�‰�‰�TÕrW   r  r§   c                 ó„  ^ ^
• Uc  SnT R                   (       d  g/ m
U
U 4S jnUb3  UT R                   ;   d   eT R                   U   u  pVU" XVU5      (       d  gO7T R                   R                  5        H  u  pxUu  pVU" XVU5      (       a  M    g   SR                  T
5      n	U	(       a  UR                  SU SU	 S35        gg)	NrS   Tc                 óH  >• X:X  a  gS n[        TR                  5       H  u  pEX%:X  d  M  Un  O   [        T5      [        L a"  U(       a  U S:X  a  UTR                  U   :X  a  SnTR                  U S[        U 5       35        TR                  U S[        U5       35        g)NFr   r5   rX  r¼   T)rÅ   r  r(  rF  rd  r  rE   )ry  rz  r§   Úvar_idræ   Ú_varÚ
conditionsrA  s         €€rX   ÚgenÚ)CppKernel.codegen_conditions.<locals>.gen.
  s¡   ø€ Ø‹|ØØˆFÜ$ T§]¡]Ö3‘�Ø•;Ø�FÙñ 4ô
 �T“
œiÒ'ÞØ˜Q“JØ˜4Ÿ;™; vÑ.Ó.à�à×Ñ   T¬+°eÓ*<Ð)=Ð>Ô?à×Ñ   S¬°SÓ)9Ð(:Ð;Ô<ØrW   Fr¨  zif(r¡  r¡   )rL  r  ÚjoinrÂ   )rA  r³   r  r§   r\  ry  rz  rZ  Ú_rangeÚjoined_conditionsr[  s   `         @rX   Úcodegen_conditionsÚCppKernel.codegen_conditions"
  sÒ   ù€ ð ‰>ØˆFØ×!×!ØØˆ
ö	ð* ‰?Ø˜$×,Ñ,Ó,Ð,Ð,Ø×+Ñ+¨CÑ0‰JˆEÙ�u 3×'Ñ'Øð (ð !%× 2Ñ 2× 8Ñ 8Ö :‘�Ø#‘
�Ù˜5 t×,Ó,Ù ñ !;ð #ŸK™K¨
Ó3ÐÞØ�N‰N˜S  ¨Ð*;Ð)<¸BÐ?Ô@ØàrW   )r~  rL  rN  r^  r³  r±  rM  rW  r  rx  rU  rV  rX  rY  ra  rS  rT  r`  r_  rd  r\  rO  rb  rP  rQ  rR  rc  rq  r]  r  )r5   éÿÿÿÿr   ©NF©F©NN)>r  r  r  r  r  rŸ  r  rD   r¡  rZ  r[  r@  r°   r�   rn  rs  r   r7   r{  rÀ   Úcontextmanagerr¥  rõ   rý  r„  rp   rˆ  r  r“  r–  r™  rl   r¦  r�  r¶  r»  r	   r9   r„   rˆ   rÁ  rn   rÅ  rÒ  rß  rã  rñ  rô  rú  rÊ  r5  r9  Úpropertyr?  r&  rK  rN  r°  rµ  ra  r!  ry  rz  s   @rX   rF  rF  z  s3  ø† ñð €IØ€EØ€MØ€Fõ'1ð^ /Ø(ô'
òRIñ˜X lÑ3õ ð( ×Ññ$ó ð$ð& BCñØ—Z‘Zõð2 %§*¡*ð 2°ô 2ð	
¨u¯z©zð 	
ÀEÇLÁLô 	
ð
 e§j¡jð 
¸5¿<¹<ô 
ò
5ð:à�j‰jð:ð �j‰jð:ð ð	:ð
 ô:ð@˜ð  U§Z¡Zô ô8ò$
ð
à�; Ð#Ñ$ðð ðð ð	ð
 �{‰{ôñ:7¨h°s©mõ 7òðN Øô*$ðZ PUôò@:òx
ò
ò(
òO%òb>ð ð! ó !ó ð!ò
ð: ×Ññ
Bó ð
Bò/ò=ò ð !%Ø&*ñ	0àð0ð ˜‘ð0ð �e—l‘lÑ#÷	0ó 0rW   rF  c                   ó|  ^ • \ rS rSr\r S)U 4S jjrS\R                  S\R                  4S jr
S\R                  S\4S	 jrS\R                  S\4S
 jrS\R                  S\4S jr\R                  4S\R                  S\4S jjrS\S\R                  S\4S jr S)S\S\R                  S\R                  S\\   4S jjr   S*S\\   S\R                  S\R                  S\\   S\\\\4      S\S\\   4S jjrS\S\R                  4U 4S jjr S+S\\\4   S\S\R                  S\R                  S\4
S jjrS)S jrS rS r S\S\4S jr!S\S\R                  S\4S  jr"S! r#S" r$SSS\RJ                  4S\\R                     S#\\   S$\\R                     4S% jjr&S)U 4S& jjr'U 4S' jr(S(r)U =r*$ ),r  iU
  Nc                 óæ   >• [         TU ]  X5        [        R                  " 5       U l        U R                  (       d   eUS:”  d   S5       eX0l        X@l        XPl        U(       a  XPl        g UU l        g )Nr   z0Expect pass in Non-Zero tiling_factor explicitly)	r?  r@  r   Úpick_vec_isaÚvec_isarÓ  r  r$  Ú	num_elems)rA  r›  ra  rÓ  r  r$  rD  s         €rX   r@  ÚCppVecKernel.__init__X
  s_   ø€ ô 	‰Ñ˜Ô+Ü"×/Ò/Ó1ˆŒØ�|�|Ðˆ|Ø˜qÓ ÐTÐ"TÓTÐ Ø*ÔØ$ŒØ"ŒÞ&/˜�°]ˆ�rW   r’   rŠ  c                 ó  ^ • T R                  X5      (       a  g U 4S jUR                   5        H-  n[        U[        5      (       d   eUR                  (       d  M-    g    [        XT R                  5      nUR                  (       a  U$ S $ )Nc              3   ó¬   >#   • U  HI  n[        U[        R                  5      (       d  M$  TR                  R                  UR
                     v •  MK     g 7fr  ©r   r   r   r±  r�  rÇ   ©r*  r�  rA  s     €rX   r,  Ú5CppVecKernel._try_get_const_stride.<locals>.<genexpr>l
  s;   øé € ð 
â'�Ü˜a¤§¡×*ó )ˆD�H‰H× Ñ  §¡Ö(Ú'ùó
   ƒ#Aª*A)r“  r‘  r¢   rG   r¦   r  rÓ  r  )rA  r’   rŠ  Úindirect_varr  s   `    rX   r  Ú"CppVecKernel._try_get_const_stridei
  s|   ø€ Ø×)Ñ)¨%×9Ñ9Øô
à×'Ò'ö
ˆLô
 ˜l¬N×;Ñ;Ð;Ð;Ø×"×"Ñ"Ùñ
ô % U°T×5GÑ5GÓHˆØ×)×)ˆvÐ3¨tÐ3rW   rˆ   r¸   c                 ó¬   • [         R                  " U R                  UR                  -  S-  U R                  R                  5       -  5      nUS:¼  d   eU$ )Né   r5   )Úmathr  rÓ  Úitemsizerl  Ú	bit_width©rA  rˆ   Únum_vectorss      rX   r%  ÚCppVecKernel._get_num_vectorsw
  sO   € Ü—i’iØ×Ñ §¡Ñ/°!Ñ3°d·l±l×6LÑ6LÓ6NÑNó
ˆð ˜aÓÐÐØÐrW   c                 óp   • U R                   UR                  -  S-  U R                  R                  5       -  $ )Nrx  )rÓ  rz  rl  r{  )rA  rˆ   s     rX   rÒ  Ú!CppVecKernel._get_raw_num_vectors~
  s0   € ð ×!Ñ! E§N¡NÑ2°QÑ6¸¿¹×9OÑ9OÓ9QÑQÐQrW   c                 ój   • U R                  U5      nUS:X  a  S[        U    S3$ S[        U    SU S3$ )Nr5   r  r�   r  r.  )r%  rH   r|  s      rX   r  ÚCppVecKernel._get_vec_typeƒ
  sJ   € Ø×+Ñ+¨EÓ2ˆØ˜!ÓØ)¬,°uÑ*=Ð)>¸aÐ@Ð@à*¬<¸Ñ+>Ð*?¸qÀÀÈQÐOÐOrW   c                 ón   • U[         R                  :X  a  gU R                  U5      nS[        U    SU S3$ )NrS   r-  r.  r�   )r„   rl   r%  rH   r|  s      rX   rC  ÚCppVecKernel._get_mask_typeŠ
  s<   € Ø”E—J‘JÓØØ×+Ñ+¨EÓ2ˆØ"¤<°Ñ#6Ð"7°q¸¸ÀQÐGÐGrW   r   c                 ó¤   • UR                   [        R                  :X  d   [        U5      5       eU R	                  U5      nU S[
        U    SU S3$ )Nz.template cast<r.  r‚   )rˆ   r„   rl   Úreprr%  rH   )rA  r   rˆ   r}  s       rX   rê  ÚCppVecKernel._get_mask_cast�
  sP   € Ø�z‰zœUŸZ™ZÓ'Ð3¬¨d«Ó3Ð'Ø×+Ñ+¨EÓ2ˆØ��¤|°EÑ':Ð&;¸1¸[¸MÈÐMÐMrW   r§   Ú	load_maskc                 ó4  • [         U   nU R                  U5      nSnU(       aX  UR                  (       d&  U R                  [        R
                  5       SU S3nO!U R                  U[        R
                  5       nUS:w  a  U S[        U5       3OUnU[        R                  :X  a.  U R                  5        SU S[        U R                  5       S3n	U	$ U(       a  U SU SU S	U S3O,U R                  U5       S
U S[        U R                  5       S3n	U	$ )aÏ  
Get a load line str that loads a vector from `var` at `index` of type `dtype`.
If `load_mask` is not None, we do a masked load accordingly.
Notes on the `dtype`:
1. We always load `self.tiling_factor` number of elements regardless of the `dtype`.
   It means we load half of the vector lanes for 16-bit data types and quarter of the
   vector lanes for 8-bit data types.
2. `torch.bool` and `torch.uint8` could mean masks and we load them as float mask vectors.
Nr  r–   r   ré   rœ   z.template loadu<r.  r½  ú::loadu()rH   r%  r¦   rC  r„   ro   rê  rE   rl   rm  r  )
rA  r§   r’   rˆ   rˆ  Úcpp_typer}  Úload_mask_strÚloadbufrÈ   s
             rX   Ú_get_vec_load_lineÚCppVecKernel._get_vec_load_line•
  s*  € ô    Ñ&ˆØ×+Ñ+¨EÓ2ˆØˆÞØ×#×#à#'×#6Ñ#6´u·{±{Ó#CÐ"DÀGÈIÈ;ÐVWÐ X‘à#'×#6Ñ#6°yÄ%Ç+Á+Ó#NÐ"O�Ø5:¸a³Z�S�E˜œ[¨Ó/Ð0Ñ1ÀSˆØ”E—J‘JÓà×)Ñ)Ó+Ð,¨G°G°9¸B¼{È4Ï>É>Ó?ZÐ>[Ð[\Ð]ˆDð ˆö !ð !�/Ð!1°(°¸1¸[¸MÈÈGÈ9ÐTUÑVà×*Ñ*¨5Ó1Ð2°(¸7¸)À2ÄkÐRV×R`ÑR`ÓFaÐEbÐbcÐdð ð
 ˆrW   rã   Ústore_valueÚ
accu_storec                 ó0	  ^ ^^^• U(       a
  Uc   S5       eU(       a	  U(       d   eTc  T R                   mS[        R                  S[        4U 4S jjmS[        R                  S[        4U 4S jjmS[        S[        4UUUU 4S jjn[        5       nUR                  S	5        UR                  5          T" U5      n	T" U5      n
S
[        U    SU
 S3nUR                  U5        U(       a   UR                  U S[        U	5       S35        [        T R                  T R                      S35      n0 nU 4S jUR                   5        H?  n[        U[        5      (       d   eUR                  (       d  M-  U" U5      nU SU S3XÞ'   MA     T R!                  UT R                  US9nSnT R"                  b{  U(       a   S5       e[        T R"                  [        5      (       d   T R"                  5       eT R"                  R                  (       a  T R"                   SU S3nOT R"                   S3n[$        R&                  " 5       (       a  UR                  ST R(                   35        OUR                  ST R(                   35        UR                  SU S3U S[        T R*                  5       S3-   U S3-   5        UR                  5          [,        R.                  " 5        n[        U5      nU H$  n[0        R2                  " SU -   S-   XÞ   U5      nM&     Ub  U SU S3OU nU(       a4  UR                  S U S35        UR5                  UR                  5       5        U(       a&  U(       a  S!OS"nUR                  U S#U S$U S%35        OUR                  S&U S'U S(35        SSS5        SSS5        U(       d(  T R7                  S)S*U5      nUR                  S+U S(35        SSS5        UR                  S,5        U(       a#  UR                  S(5        TR9                  U5        gT R:                  R=                  TXƒS-9n[        U[        5      (       d   eS.Ul        U$ ! , (       d  f       NÂ= f! , (       d  f       NË= f! , (       d  f       N¥= f)/a  
Load or store a vector in a non-contiguous way. The vector is initialized from an array that is
filled in an inner loop over the tiling factor.
:param var: buffer to load from or store to, i.e. `var[transformed(index)]`. If None, we load the index
            as index expression, i.e. `transformed(index)`.
:param index: index into the `var` or the index expression by its own if `var` is None.
              The `index` could contain indirect indexing or the tiling itervar. When used in
              the inner loop, the index is transformed as follows:
              1. the index is linearized along the tiling dim.
              2. the indirect indexing vector variables are transformed into arrays over the tiling dim.
:param dtype: data type of `var` or `index` if `var` is None.
:param buffer: the code buffer to write the generated code to. If None, we write to `self.loads`.
:param store_value: the value to store. If None, we load the vector.
:param accu_store: whether accumulate the store_value to store_ptr. If True, a store_value should be provided
:return: a CppCSEVariable that represents the loaded vector or None if it is a store.
Nzstore var must be providedrˆ   r¸   c                 ót   >• U R                   S:  a  TR                  SU R                   -  -  $ TR                  $ ©Né   )rz  rm  ©rˆ   rA  s    €rX   Úget_result_sizeÚCCppVecKernel._load_or_store_non_contiguous.<locals>.get_result_sizeÙ
  s1   ø€ Ø�~‰~ Ó!Ø—~‘~¨¨e¯n©nÑ)<Ñ=Ð=à—~‘~Ð%rW   c                 ót   >• U R                   S:  a  TR                  SU R                   -  -  $ TR                  $ r”  )rz  rÓ  r–  s    €rX   Úget_tiling_sizeÚCCppVecKernel._load_or_store_non_contiguous.<locals>.get_tiling_sizeß
  s5   ø€ Ø�~‰~ Ó!Ø×)Ñ)¨Q°%·.±.Ñ-@ÑAÐAà×)Ñ)Ð)rW   Úvec_varc                 ón  >• U R                   (       d   e[        5       nUR                  S5        UR                  5          U R                  nUc   eU[
        R                  :X  a  [
        R                  nT" U5      nT	" U5      nUR                  S[        U    SU S35        U  S[        U5       S3nUR                  U5        UR                  S5        S S S 5        UR                  S5        T
R                  R                  TU5      n[        U[        5      (       d   eU$ ! , (       d  f       NT= f)	Nr  r!  rœ   r"  r#  rÖ   zreturn tmpbuf;rÒ  )r¦   r7   rÂ   rÄ   rˆ   r„   rl   ro   rH   rE   r±  r²  r¢   rG   )rœ  r³   r/  Úresult_sizeÚtiling_sizerÈ   rº  rã   r—  rš  rA  s          €€€€rX   Úvec_to_arrayÚ@CppVecKernel._load_or_store_non_contiguous.<locals>.vec_to_arrayå
  s  ø€ Ø—>—>Ð!�>Ü“>ˆDØ�N‰N˜5Ô!Ø—‘•Ø#ŸM™M�	Ø Ñ,Ð,Ð,Ø¤§
¡
Ó*Ü %§¡�IÙ-¨iÓ8�Ù-¨iÓ8�Ø—‘Ø.¬|¸IÑ/FÐ.GÀrÈ+ÈÐV_Ð`ôð "˜Ð"8¼À[Ó9QÐ8RÐRTÐU�Ø—‘˜tÔ$Ø—‘Ð/Ô0÷ ð �N‰N˜4Ô Ø—X‘X×&Ñ& v¨tÓ4ˆFÜ˜f¤n×5Ñ5Ð5Ð5ØˆM÷! •ús   Á BD&Ä&
D4r  r!  rœ   r"  r#  rÖ   rZ  c              3   ó¬   >#   • U  HI  n[        U[        R                  5      (       d  M$  TR                  R                  UR
                     v •  MK     g 7fr  rq  rr  s     €rX   r,  Ú=CppVecKernel._load_or_store_non_contiguous.<locals>.<genexpr>  s;   øé € ð !â+�AÜ! !¤T§X¡X×.ó -�—‘×$Ñ$ Q§V¡VÖ,Ú+ùrt  r©  rª  ©rƒ  rP  zunexpected store with load maskz.is_masked(r–   z != 0z#pragma GCC unroll z#pragma unroll ú
for (long ú = 0; r¼   r½   r¾   r¿   r  ú+=Ú=r˜   z tmpbuf[rØ   ztmpbuf[r±  r»   ztmpbuf.data()r   rž  rÒ  r  T)rx  r„   rˆ   rn   rG   r7   rÂ   rÄ   rH   rE   r-   r  r  r‘  r¢   r¦   r„  r~  r   Úis_gccrÓ  rm  rÀ   rÁ   rÉ   rÊ   rÃ   rŽ  r\  r±  r²  )rA  r§   r’   rˆ   rã   r�  r‘  r   r³   rž  rŸ  Úresult_declareÚitervar_innerÚreplacementsru  Ú	array_varrˆ  rÌ   Úindex_cÚrhsr¬   Ú	load_linerº  r—  rš  s   `   `                  @@rX   r  Ú*CppVecKernel._load_or_store_non_contiguousº
  s8  û€ ö2  #¡/ÐOÐ3OÓOÐ1ÞÞÐ�;Ø‰>Ø—Z‘ZˆFð	&¤5§;¡;ð 	&´3÷ 	&ð	*¤5§;¡;ð 	*´3÷ 	*ð	¤.ð 	´^÷ 	ò 	ô, ‹~ˆØ�‰�uÔØ�[‰[�]Ù)¨%Ó0ˆKÙ)¨%Ó0ˆKà*¬<¸Ñ+>Ð*?¸rÀ+ÀÈiÐXð ð �N‰N˜>Ô*ÞØ—‘Ø"�mÐ#9¼+ÀkÓ:RÐ9SÐSUÐVôô /Ø—=‘= §¡Ñ1Ð2°&Ð9óˆMð ˆLô!à×+Ò+ö!�ô
 " ,´×?Ñ?Ð?Ð?Ø×&×&Ñ&Ù ,¨\Ó :�IØ4=°;¸aÀ¸ÈaÐ1P�LÓ.ñ!ð ×0Ñ0Ø 4§?¡?¸=ð 1ð ˆEð ˆIØ�‰Ñ*Þ&ÐIÐ(IÓI�Ü! $§/¡/´>×BÑBÐSÀDÇOÁOÓSÐBØ—?‘?×)×)Ø#'§?¡?Ð"3°;¸}¸oÈQÐ O‘Ià#'§?¡?Ð"3°5Ð 9�IÜ×!Ò!×#Ñ#Ø—‘Ð!4°T×5GÑ5GÐ4HÐIÕJà—‘ °×1CÑ1CÐ0DÐEÔFØ�N‰NØ˜]˜O¨6Ð2Ø"�O 3¤{°4·>±>Ó'BÐ&CÀ2ÐFñGà"�O 3Ð'ñ(ôð
 —‘•¤
× 4Ò 4Ô 6¸%Ü% eÓ,�ã$0�LÜ ŸfšfØ < .Ñ1°EÑ9Ø$Ñ2Øó’Gñ %1ð .1©_˜˜˜Q˜w˜i qÑ)ÀWÀI�ÞØ—N‘N T¨)¨°AÐ#6Ô7Ø×'Ñ'¨¯©«Ô6ÞÞ*4¡$¸#�KØ—N‘N c U¨!¨K¨=¸ÀÀÈrÐ#RÕSà—N‘N W¨]¨O¸4À¸uÀAÐ#FÔG÷# !7—ö$ Ø ×3Ñ3°OÀQÈÓN�	Ø—‘ ¨¨°1Ð5Ô6÷A ðB 	�‰�tÔÞØ�N‰N˜3ÔØ�M‰M˜$ÔØà—X‘X×&Ñ& v¨tÐ&ÐAˆFÜ˜f¤n×5Ñ5Ð5Ð5Ø ˆFŒMØˆM÷= !7Õ 6ú—•ú÷Y �]úsE   Â3B?RÅ6ERËQ6Ë%CQ%Î)Q6Î17RÑ%
Q3Ñ/Q6Ñ6
R	Ò RÒ
RrÇ   c                 ó4  >• U R                   R                  U5      nU R                  U5      n[        R                  R                  U5      nU R                  U R                     nU R                  X%5      nUS:X  a  [        T	U ])  X5      $ US:X  aA  U R                  X2X@R                  5      nU R                  R                  U R                  XtS9nOU R!                  X2U5      n[#        U[$        5      (       d   eUR'                  SXU40 5        SUl        U$ )Nr   r5   r  r�  T)r›  r«  r˜  r4   r¬  r­  r  r  r  r?  r�  rŽ  r~  r±  r²  rx  r  r¢   rG   r´  r¦   )
rA  rÇ   r’   r§   rˆ   r  r  rÈ   rº  rD  s
            €rX   r�  ÚCppVecKernel.loadI  só   ø€ Ø�i‰i�o‰o˜dÓ#ˆØ×$Ñ$ UÓ+ˆÜ—‘×!Ñ! $Ó'ˆØ—]‘] 4§?¡?Ñ3ˆ
Ø×+Ñ+¨EÓ>ˆØ�Q‹;ä‘7‘< Ó,Ð,Ø�q‹[à×*Ñ*¨3°u¿o¹oÓNˆDØ—X‘X×&Ñ& t§z¡z°4Ð&ÐE‰Fà×7Ñ7¸ÀEÓJˆFÜ˜&¤.×1Ñ1Ð1Ð1Ø×Ñ˜f t°5Ð&9¸2Ô>ØˆŒØˆrW   rï   c           	      óþ  • [        U[        5      (       d-  [        U[        5      (       a  UR                  (       d   U5       eU R                  U R
                     nU S[        U5       3nU R                  X65      n[        5       n	US:X  aß  U(       an  U[        R                  :X  a$  U R                  c  U R                  U5       SU S3O,U R                  U5       SU S[        U R                  5       S3n
SU SU
 S3nU[        R                  :X  a&  U R                  c  U	R                  U SU S35        U	$ U	R                  U SU S[        U R                  5       S35         U	$ U R                  X#XIXS	9  U	$ )
a  
Get a store line buffer that stores `value` into `var` at `index` of `dtype`. It handles
both contiguous and non-contiguous store cases.
:param value: Vectorized type templaterized on `dtype`.
:param var: buffer to store into.
:index: index into the `var`.
ré   r5   rŠ  r–   rœ   r¡  ú.store(rÖ   )rã   r�  r‘  )r¢   rp   rG   r¦   r  r  rE   r  r=   r„   ro   r$  r  rm  rÂ   r  )rA  rï   r§   r’   rˆ   r‘  r  Úvar_exprr  r³   r�  s              rX   Ú_get_store_lineÚCppVecKernel._get_store_line]  s�  € ô" ˜%¤×%Ñ%Ü�uœn×-Ñ-°%·,·,ð	àó	ð 
ð —]‘] 4§?¡?Ñ3ˆ
Ø�U˜#œk¨%Ó0Ð1Ð2ˆØ×+Ñ+¨EÓ>ˆÜÓˆØ�Q‹;Þð ¤§¡Ó+°·±Ñ0Fð ×)Ñ)¨%Ó0Ð1°¸(¸À1ÑEà ×.Ñ.¨uÓ5Ð6°h¸x¸jÈÌ;ÐW[×WeÑWeÓKfÐJgÐghÐið ð
 ˜E˜7 # d V¨1Ð-�ØœŸ™Ó#¨¯©Ñ(>Ø—‘ % ¨°¨z¸Ð<Ô=ð ˆð —‘Ø�g˜W X J¨b´¸T¿^¹^Ó1LÐ0MÈRÐPõð ˆð ×.Ñ.Ø˜E¸Eð /ñ ð ˆrW   c                 óÌ  ^• ST;   d   e[        U[        5      (       d   U5       eUR                  (       d  U R                  U5      nU R                  R                  T5      nU R                  U5      n[        R                  R                  T5      nUcA  U R                  X5X&5      nU R                  R                  UR                  U4S j5      5        g US:X  Gaz  [        R                  R                   (       dS  U R"                  S:X  aC  U R                  U UUUSS9nU R                  R                  UR                  U4S j5      5        g U R%                  U5      nU R%                  [&        R(                  5      n	[*        U   n
[,        R.                  " U[&        R(                  5      R0                  n[        U[        5      (       a  UR                  (       d   eU R2                  (       a,  SU
 S	U	 S	U S
U S	U S	U S	[5        U R2                  5       S3nOSU
 S	U	 S	U S
U S	U S	U S3nU R                  R7                  [9        TU5      5        g [;        SU 35      e)Nr°  c                 ó   >• [        TU 5      $ r  ©r;   ©r·  rÇ   s    €rX   Ú<lambda>Ú$CppVecKernel.store.<locals>.<lambda>”  s   ø€ ´,¸tÀQÔ2GrW   r²  r5   T)r‘  c                 ó   >• [        TU 5      $ r  r»  r¼  s    €rX   r½  r¾  ž  s   ø€ ´lÀ4ÈÔ6KrW   zatomic_add_vec<rœ   r½  rÖ   r³  )r¢   rG   r¦   r  r›  r´  r˜  r4   r¬  r­  r·  rq  r\  Úmapr   rð  ri  ra  r%  r„   r  rH   r2   rš  rï   r$  rE   rÂ   r;   rø  )rA  rÇ   r’   rï   rµ  r§   rˆ   r³   Ún_srcÚn_idxr‰   rÈ   s    `          rX   r¶  ÚCppVecKernel.store‰  s  ø€ Ø˜‹}Ðˆ}Ü˜%¤×0Ñ0Ð7°%Ó7Ð0Ø�|�|à—N‘N 5Ó)ˆEØ�i‰i×Ñ˜tÓ$ˆØ×$Ñ$ UÓ+ˆÜ—‘×!Ñ! $Ó'ˆØ‰<Ø×'Ñ'¨°EÓAˆDØ�K‰K×Ñ˜tŸx™xÔ(GÓHÕIØ�\Ô!Ü—:‘:×-×-°$×2BÑ2BÀaÓ2GØ×+Ñ+Ø�gØØØØ#ð ,ð �ð —‘×"Ñ" 4§8¡8Ô,KÓ#LÕMà×-Ñ-¨eÓ4�Ø×-Ñ-¬e¯k©kÓ:�Ü% eÑ,�ÜŸš u¬e¯k©kÓ:×@Ñ@�Ü! %¬×8Ñ8¸U¿\¿\ÐIÐIØ—>—>Ø,¨V¨H°B°u°g¸RÀ¸wÀbÈÈÈRÐPUÈwÐVXÐY^ÐX_Ð_aÔbmÐnr×n|Ñn|Ób}Ða~ð  Að  B‘Dà,¨V¨H°B°u°g¸RÀ¸wÀbÈÈÈRÐPUÈwÐVXÐY^ÐX_Ð_aÐb�DØ—‘×%Ñ%¤l°4¸Ó&>Õ?ä%¨°D°6Ð&:Ó;Ð;rW   c                 óô  • U[         ;   d   eUS;   nU R                  U R                  :¬  nU(       a  UOUn[        U[        5      (       d   U5       eUR
                  (       d  U R                  U5      nX#U4nX€R                  R                  ;   a  U R                  R                  U   $ Sn	U	 S[        U    S3n
[        X75      nU R                  X75      nU R                  R                  U R                  SU 3SS9n[        U[        5      (       d   eU S3nS	U 3nS	U 3nU =R                  U UU/-  sl        S
U l        U R                   R#                  U R%                  XÛX7[&        5      5        U R                   R#                  U R%                  UUUUU R(                  5      5        U R+                  X1S5      nU(       Gaî  U R                   R#                  U R%                  UUUUU R(                  5      5        U R                  c   e[,        R.                  " [0        R2                  U R4                  U R                  S 5      nUS:X  a)  U R6                  R                  U R8                  SU 3SS9nO(U R:                  R                  U R8                  SU 3SS9nS	U 3n[=        U R4                  U R                     U R>                  5      (       am  U R                  U R                  :¼  aQ  [=        UU R4                  U R                     5      [=        U R4                  U R                     U R>                  5      -  OUO[@        RB                  " S5      nU R4                  U R                     U R>                  -  (       a?  U R                  U R                  :¼  a#  [=        UU R4                  U R                     5      OUO[@        RB                  " S5      nSU 3nU RE                  UUUUUS
S9  U RE                  X=UUU5        U RE                  UUUUU5        U RF                  (       a  UOUnU RF                  (       a  UOUnUS:X  a4  U RH                  RK                  U SU RM                  UUUU5       S35        OêU RH                  RK                  U SU RM                  UUUU5       S35        O¶U R                  c   eU RN                  U R                     n[Q        U R                  S-   [S        U RN                  5      5       H%  nUU R4                  U   -  U RN                  U   -   nM'     UUUUS.nU RH                  RK                  U SU RL                  " X>40 UD6 S35        U RU                  UUUUU RL                  U R(                  S9  U RU                  UUUU[V        [&        S9  U(       a(  U RU                  UUUUU RL                  U R(                  S9  U[X        RZ                  :H  nU(       Ga^  []        U5      (       aT  U R_                  U5      S;   d   S5       eSU S3nSU S3nU R`                  RK                  U S[W        X=U5       S35        OÍU(       a	  U SU S3nO½U(       a  US;   a  SU S3nO©US:X  d   eU S3nO›S U RM                  US!S"5      -   S#-   n U[X        RZ                  :H  nU(       a  [X        Rb                  OUn!S$[        U!    S3n
S%[        U!    S&U R_                  U!5       S3n"U n#U(       a  US:X  d   eU S'U 3n#U" S(U
 S)U
 S*U  S&U# S3
nU R`                  RK                  U S[W        X=UUS+9 S35        Un$O|Un$[]        U5      (       a2  S	U$ 3n%U R`                  RK                  U$ S[W        UU$U%5       S35        O8U(       a1  US:X  d   eS	U$ 3n%U R`                  RK                  U$ SU$ S'U% S35        [e        UU$5      n&U&U R                  R                  U'   U&$ ),a  
Perform vectorized reduction operation.

This method handles vectorized reduction for different reduction types.
It manages special cases for low-precision floating point types and
employs precision improvement techniques for certain reduction operations.

Args:
    dtype: The output data type for the reduction result
    src_dtype: The source data type of the input value
    reduction_type: Type of reduction operation (sum, min, max, etc.)
    value: The input value to reduce

Returns:
    The result of the reduction operation
ræ  zat::vecz::Vectorized<r�   rç  Frè  râ  Úmasked_TNrh   r   rê  rë  rd   rº   r»   r5   )r¨   r’   Úhorizontal_reductionrª   )rj  rk  )r5   r   z4Welford reduction does not support VectorizedN (N>2)zwelford_vec_reduce_all(r–   z_vec_reduce_all()rg   rd   r`   r­  z.all_zero()r_   z.all_masked()z	{ return r·  r=  z; }r  zat::vec::vec_reduce_all<rœ   ré   z([](z& x, z& y) rh  )3ÚVECTORIZABLE_RTYPESr  rO  r¢   rG   r¦   r  r\  rì  rH   r‘   Úreduction_acc_type_vecr²  rx  rc  rW  rQ  r  rÁ  r�   Úreduction_init_vecrÒ  r`  ra  rb  rc  rd  r]  r³  r^  r   rÓ  rõ   re  rã  r$  rq  rÂ   Úreduction_combine_vecr  r^  rÓ   rn  r°   r„   rl   r*   r%  rR  ro   r¤   )'rA  rˆ   rª   r‡   rï   rí  rÆ  rï  rî  Úvec_nsÚvecrÒ   Úacc_type_vecr²   Úacc_vecÚ
masked_accÚmasked_acc_vecÚuse_acc_helperrÏ  r©   Úmasked_helper_valÚhelper_vec_rangeÚmasked_helper_vec_rangerð  Úacc_vec_Úhelper_val_r’   ræ   r	  r«   r¨   Úmasked_next_valueÚreduce_all_bodyr/  Úvec_reduce_all_funcÚ
result_vecÚtmpvarÚmasked_tmpvarr  s'                                          rX   rñ  ÚCppVecKernel.reduction­  s³  € ð& Ô!4Ó4Ð4Ð4Ø)Ð-AÑAÐØ#Ÿ™°$×2FÑ2FÑFÐÞ"2‘Y¸ˆ
Ü˜%¤×0Ñ0Ð7°%Ó7Ð0à�|�|Ø—N‘N 5Ó)ˆEà!°5Ð8ˆØ×.Ñ.×>Ñ>Ó>Ø×%Ñ%×5Ñ5°mÑDÐDàˆØ�˜¤l°5Ñ&9Ð%:¸!Ð<ˆÜ% nÓAˆØ×2Ñ2°>ÓNˆà× Ñ ×)Ñ)Ø�J‰J˜* ] OÐ4¸Eð *ð 
ˆô ˜#œ~×.Ñ.Ð.Ð.Ø�E˜�,ˆØ˜s˜e�_ˆ
Ø" 7 )Ð,ˆØ× Ò ¨ u¨w¸Ð$GÑGÕ Ø ˆÔØ×(Ñ(×/Ñ/Ø×&Ñ&Ø˜~¼>óô	
ð
 	×(Ñ(×/Ñ/Ø×&Ñ&ØØØØØ×'Ñ'óô	
ð ×1Ñ1°.ÈÓOˆßà×,Ñ,×3Ñ3Ø×*Ñ*Ø"Ø Ø"ØØ×+Ñ+óôð ×'Ñ'Ñ3Ð3Ð3Ü&×-Ò-Ü—‘˜dŸk™k¨$×*>Ñ*>Ð*@ÐAóˆNð Ð!1Ó1Ø!×4Ñ4×=Ñ=Ø—L‘L J¨}¨oÐ">Àeð >ð ‘
ð "×4Ñ4×=Ñ=Ø—L‘L J¨}¨oÐ">Àeð >ð �
ð #*¨*¨Ð 6Ðô ˜DŸK™K¨¯©Ñ8¸$×:LÑ:L×MÑMð —‘¨$×*>Ñ*>Ó>ô ˜^¨T¯[©[¸¿¹Ñ-IÓJÜ˜tŸ{™{¨4¯?©?Ñ;¸T×=OÑ=OÓPòQñ (ô —]’] 1Ó%ð ð  —;‘;˜tŸ™Ñ/°$×2DÑ2D×Dð —‘¨$×*>Ñ*>Ó>ô ˜^¨T¯[©[¸¿¹Ñ-IÔJá'ô —]’] 1Ó%ð $ð #*¨*¨Ð 6ÐØ× Ñ ØØØ!ØØØð !ñ ð × Ñ Ø ZÐ1AÀ5ôð × Ñ ØØØ!Ø'Øôð *.¯¯‘~¸WˆHØ/3¯~¯~Ñ+À:ˆKØ Ó&Ø—‘×%Ñ%Ø�j  D×$>Ñ$>¸~ÈxÐY^Ð`kÓ$lÐ#mÐmnÐoõð —‘×%Ñ%Ø�j  D×$>Ñ$>¸~ÈxÐY^Ð`kÓ$lÐ#mÐmnÐoõð ×'Ñ'Ñ3Ð3Ð3Ø—M‘M $×"6Ñ"6Ñ7ˆEÜ˜4×/Ñ/°!Ñ3´S¸¿¹Ó5GÖH�Ø §¡¨A¡Ñ.°·±¸qÑ1AÑA’ñ Ið $ØØ(<Ø&ñ	ˆFð �K‰K×!Ñ!Ø�)˜3˜t×9Ò9¸.Ñ\ÐU[Ñ\Ð]Ð]^Ð_ôð 	×,Ñ,ØØØØØ!%×!;Ñ!;Ø"×5Ñ5ð 	-ñ 	
ð 	×,Ñ,ØØØØÜ!2Ü,ð 	-ñ 	
ö à×0Ñ0ØØØØØ%)×%?Ñ%?Ø"&×"9Ñ"9ð 1ñ ð œ5Ÿ:™:Ñ%ˆßä# N×3Ñ3Ø×,Ñ,¨UÓ3ð 8ó ð Jð JóJð ð  7°w°i¸qÐA�
Ø&=¸nÐ=MÈQÐ$OÐ!Ø×%Ñ%×/Ñ/Ø�e˜3Ô0°ÐFWÓXÐYÐYZÐ[õö "Ø .Ð/Ð/?À¸yÈÐJ‘
ÞØ!ð &ó ð
 $% W I¨[Ð!9‘Jà)¨UÓ2Ð2Ð2Ø$+ 9¨MÐ!:‘Jð  Ø×0Ñ0°ÀÀcÓJñKàñð  ð
  ¤5§:¡:Ñ-�æ+2œEŸKšK¸�	Ø,¬\¸)Ñ-DÐ,EÀQÐG�Ø(@ÄÈiÑAXÐ@YÐY[Ð\`×\qÑ\qÐr{Ó\|Ð[}Ð}~Ð&Ð#Ø '˜y�
Þ!Ø)¨UÓ2Ð2Ð2Ø$+ 9¨C°Ð/?Ð!@�JØ 3Ð4°D¸¸¸UÀ3À%ÀuÈ_ÐL]Ð]_Ð`jÐ_kÐklÐm�
à×!Ñ!×+Ñ+Ø�%�sÔ,¨^À*ÐXaÑbÐcÐcdÐeôð ‰FàˆFÜ# N×3Ñ3Ø")¨&¨Ð 2�Ø×%Ñ%×/Ñ/Ø�h˜cÔ"3°NÀFÈMÓ"ZÐ![Ð[\Ð]õö  Ø%¨Ó.Ð.Ð.Ø")¨&¨Ð 2�Ø×%Ñ%×/Ñ/Ø�h˜c & ¨¨]¨O¸1Ð=ôô # >°6Ó:ˆØ<Bˆ×Ñ×*Ñ*¨=Ñ9ØˆrW   c                 ó8  ^• U R                  U5      nU R                  R                  T5      n[        R                  R                  T5      nUR                  (       a%  U[        R                  :w  a  [        R                  nOUn[        R                  R                  U5      n[        R                  R                  U5      n[        5       n	U R                  U R                  :¼  a.  U	R                  U S[!        U5       S["        U    SU S35        OØXV:w  a²  ["        U   R%                  SS5       SU 3n
U[        R&                  :X  a&  U SU R                  [        R&                  5       S3nO@X‡s=:X  a  S	:X  a  O  OS
["        U    SU S3nO S
["        U    SU S["        U    SU SU S3nU	R                  SU
 SU S35        U
nU	R)                  U R+                  X4X%5      5        U R,                  R)                  U	R/                  U4S j5      5        g )Nr©  z] = static_cast<r½  rÖ   z::rÍ   z.template cast<bool,r‚   r5   úat::vec::convert<r–   r.  rÕ   rº   r»   c                 ó   >• [        TU 5      $ r  r»  r¼  s    €rX   r½  Ú.CppVecKernel.store_reduction.<locals>.<lambda>Æ  s   ø€ ¼ÀTÈ1Ô8MrW   )r˜  r›  r´  r4   r¬  r­  Úis_floating_pointr„   rk   ro   rr  r%  r=   r  rO  rÂ   rE   rH   r  rl   r\  r·  rR  rÀ  )rA  rÇ   r’   rï   r§   Ú	out_dtyperˆ   Úout_num_vectorsÚsrc_num_vectorsr³   Úconverted_valueÚconverts    `          rX   rô  ÚCppVecKernel.store_reduction¡  sÿ  ø€ Ø×$Ñ$ UÓ+ˆØ�i‰i×Ñ˜tÓ$ˆÜ—G‘G×%Ñ% dÓ+ˆ	Ø×&×&¨9¼¿¹Ó+DÜ—K‘K‰EàˆEÜŸ(™(×3Ñ3°IÓ>ˆÜŸ(™(×3Ñ3°EÓ:ˆÜÓˆØ�?‰?˜d×2Ñ2Ó2à�N‰NØ�%�qœ UÓ+Ð,Ð,<¼\È)Ñ=TÐ<UÐUWÐX]ÐW^Ð^`Ðaõð
 Ó!ä# IÑ.×6Ñ6°t¸SÓAÐBÀ!ÀEÀ7ÐKð  ð ¤§
¡
Ó*Ø!& Ð';¸D×<QÑ<QÔRW×R\ÑR\Ó<]Ð;^Ð^aÐb‘Gà&Õ>¸QÖ>à/´¸YÑ0GÐ/HÈÈ5È'ÐQRÐSñ  ð
 0´¸YÑ0GÐ/HÈØ.Ð/¨q´¸eÑ1DÐ0EÀQÀÐFWÐWYÐZ_ÐY`Ð`aðcð  ð —‘  Ð&7°s¸7¸)À1ÐEÔFØ'�Ø�K‰K˜×,Ñ,¨U¸ÓJÔKØ×Ñ×$Ñ$ T§X¡XÔ.MÓ%NÕOrW   Ú
scalar_varc                 ó.  • UR                   (       a   eUR                  [        R                  :X  aE  U R                  R                  U R                  U R                  5        SUR                   S35      nO^UR                  c   eU R                  R                  U R                  U R                  UR                  5       SUR                   S35      n[        U[        5      (       d   eUR                  Ul        UR                  Ul        SUl         U$ )Nr  r–   r¡  T)r¦   rˆ   r„   rl   r±  r²  r³  rC  rÇ   r  r¢   rG   Údependent_itervars)rA  ré  rœ  s      rX   r  ÚCppVecKernel.broadcastÈ  sî   € Ø×$×$Ð$Ð$Ø×ÑœuŸz™zÓ)Ø—h‘h×'Ñ'Ø—‘ ×!4Ñ!4Ó!6Ð 7°w¸z¿¹Ð>OÈqÐQó‰Gð ×#Ñ#Ñ/Ð/Ð/Ø—h‘h×'Ñ'Ø—‘Ø×%Ñ% j×&6Ñ&6Ó7Ð8¸¸*¿/¹/Ð9JÈ!ÐLóˆGô ˜'¤>×2Ñ2Ð2Ð2Ø"×(Ñ(ˆŒØ%/×%BÑ%BˆÔ"ØˆŒØˆrW   r  c           	      ó8  • UR                   (       a   eUR                  c   eU R                  R                  U R                  U R                  UR                  5       SU SU S35      n[        U[        5      (       d   eUR                  Ul        SUl         U$ )Nz	::arange(rœ   r–   T)r¦   rˆ   r±  r²  r³  r  r¢   rG   )rA  r’   r  rº  s       rX   r  ÚCppVecKernel.arangeÚ  sŒ   € Ø—<—<ÐÐØ�{‰{Ñ&Ð&Ð&Ø—‘×"Ñ"Ø�L‰LØ×!Ñ! %§+¡+Ó.Ð/¨y¸¸¸rÀ&ÀÈÐKó
ˆô ˜&¤.×1Ñ1Ð1Ð1Ø—{‘{ˆŒØˆŒØˆrW   c                 óð  • [         U   nU R                  U5      n[        U5      (       a  SU S3$ US;   aa  [        U   nU R	                  X5      nUS:X  a  [        U5      (       a  SU S3OSU S3nO[        U5      (       a  SU S3OSU S	3nU S
U S3$ US:X  a  U R                  5        S3$ [        X5      nU S
U S3n	U[        R                  :X  a  US;   d   eU R                  5        SU S3$ U	$ )Nr�   r‚   ry   ra   r|   r{   r~   rz   r}   r¡  r–   rg   z	::from(0))r_   r`   rd   r  )
r<   r  r*   rH   rÈ  r   rC  r�   r„   rl   )
rA  r‡   rˆ   r�   Úvec_typer‰   rÒ   r’  Úscalar_initÚvec_inits
             rX   rÉ  ÚCppVecKernel.reduction_init_vecæ  sF  € Ü0°Ñ7ˆØ×%Ñ% kÓ2ˆä ×/Ñ/Ø˜h˜Z sÐ+Ð+àÐ1Ó1Ü! +Ñ.ˆFØ×2Ñ2°>ÓIˆHØ Ó)ô & e×,Ñ,ð +¨6¨(°-Ñ@à/°¨x°xÐ@ñ ô & e×,Ñ,ð ,¨F¨8°=ÑAà/°¨x°xÐ@ð ð
 �Z˜q   QÐ'Ð'à˜UÓ"Ø×)Ñ)Ó+Ð,¨IÐ6Ð6ä$ ^Ó;ˆØ�Z˜q  ¨QÐ/ˆØ”E—J‘JÓØ!Ð%:Ó:Ð:Ð:Ø×)Ñ)Ó+Ð,¨G°K°=ÀÐBÐBØˆrW   c                 ó¼  • [         U   nU R                  U5      n[        U5      (       a  SU S3$ US;   ax  U R                  U5      nU R                  [        R
                  5      nU[        R                  :X  a!  S[        [        R                      SU SU S3$ S[        U    SU SU S3$ U[        R                  :X  a  US;   d   eU R                  5        $ U$ )Nr�   r�   ry   zIndexValueVec<rœ   )r_   r`   rg   rd   )
r<   r  r*   r%  r„   r  rl   rH   ro   rC  )rA  r‡   rˆ   r�   rð  rÁ  rÂ  s          rX   rÈ  Ú#CppVecKernel.reduction_acc_type_vec  sé   € Ü0°Ñ7ˆØ×%Ñ% kÓ2ˆÜ ×/Ñ/Ø˜h˜Z qÐ)Ð)ØÐ1Ó1Ø×)Ñ)¨+Ó6ˆEØ×)Ñ)¬%¯+©+Ó6ˆEØœŸ
™
Ó"Ø'¬´U·[±[Ñ(AÐ'BÀ"ÀUÀGÈ2ÈeÈWÐTUÐVÐVØ#¤L°Ñ$=Ð#>¸bÀÀÀrÈ%ÈÐPQÐRÐRØ”E—J‘JÓØ!Ð%AÓAÐAÐAØ×)Ñ)Ó+Ð,Ð-ØˆrW   rÆ  rª   c                 ó|  • U[         R                  :H  nUS:X  aG  U R                  (       a  SU SU S[        U R                  5       S3$ U(       a  U SU 3$ SU SU S3$ US:X  aG  U R                  (       a  SU SU S[        U R                  5       S3$ U(       a  U S	U 3$ S
U SU S3$ US:X  a…  U(       a9  U R                  (       a  SU S[        U R                  5       SU S3$ SU SU S3$ U R                  (       a  SU SU S[        U R                  5       S3$ U(       a  SOSn	U SU	 SU 3$ US:X  a7  U R                  (       a  SU SU S[        U R                  5       S3$ U SU 3$ US:X  a7  U R                  (       a  SU SU S[        U R                  5       S3$ U SU 3$ US:X  a  U(       a?  U R                  (       a"  SU SU S[        U R                  5       SU S3	$ SU SU SU S3$ U R                  (       a  SU SU S[        U R                  5       S3$ SU SU S3$ US:X  an  [	        U[
        5      (       a  Uu  p«nO[        X5      u  p«nU R                  (       a%  SU SU
 SU SU S[        U R                  5       S3$ SU SU
 SU SU S3	$ US;   aí  Uc   e[        U   nU[         R                  :X  a  [        [         R                     nU R                  U5      nU R                  [         R                  5      nSnSnUb&  Uc   eS[        U5      R                  5        3nSU 3nU R                  (       a.  U S U SU SU U S!U SU U S[        U R                  5       S3$ U S U SU SU U S!U SU U S3$ US":X  a“  [	        U[        5      (       aG  UR                  [         R                  :X  d   e[        [         R"                  R$                  U45      u  nU R                  (       a  S#U SU S[        U R                  5       S3$ U SU 3$ [&        e)$Nr`   zmax_masked_reduce(rœ   r–   rÃ  rã  r_   zmin_masked_reduce(r»  rÛ  rd   r”   r•   zsum_masked_reduce(r—   r[   r˜   re   zprod_masked_reduce(r™   rf   zxor_sum_masked_reduce(rš   rh   r�   ri   rž   z}, rŸ   ry   rS   z_combine_vec<r½  rg   zany_masked_reduce()r„   rl   r$  rE   r¢   r£   r¤   rH   ro   r%  r  rp   r›  rG   rˆ   rO   r4   rr  r³  rø  )rA  r‡   r§   r¨   r©   r’   rÆ  rª   r«   r¬   r­   r®   r¯   r‰   rÁ  rÂ  Út_extraÚ	arg_extras                     rX   rÊ  Ú"CppVecKernel.reduction_combine_vec  s  € ð œuŸz™zÑ)ˆØ˜UÓ"Ø�~�~Ø+¨C¨5°°:°,¸bÄÈTÏ^É^ÓA\Ð@]Ð]^Ð_Ð_ö ð �e˜3˜z˜lÐ+ðð -¨S¨E°°J°<¸qÐAðð
 ˜uÓ$Ø�~�~Ø+¨C¨5°°:°,¸bÄÈTÏ^É^ÓA\Ð@]Ð]^Ð_Ð_ö ð �e˜3˜z˜lÐ+ðð -¨S¨E°°J°<¸qÐAðð
 ˜uÓ$ÞØ—>—>Ø1°*°¸RÄÈDÏNÉNÓ@[Ð?\Ð\_Ð`jÐ_kÐklÐmÐmà1°*°¸SÀÀÈAÐNÐNà—>—>Ø/°¨u°B°z°lÀ"Ä[ÐQU×Q_ÑQ_ÓE`ÐDaÐabÐcÐcæ)0¡#°c�KØ!˜U ! K =°°*°Ð>Ð>Ø˜vÓ%Ø�~�~Ø,¨S¨E°°J°<¸rÄ+ÈdÏnÉnÓB]ÐA^Ð^_Ð`Ð`à˜˜c * Ð.Ð.Ø˜yÓ(Ø�~�~Ø/°¨u°B°z°lÀ"Ä[ÐQU×Q_ÑQ_ÓE`ÐDaÐabÐcÐcà˜˜c * Ð.Ð.ØÐ/Ó/ÞØ—>—>Ø-¨c¨U°"°Z°LÀÄ;ÈtÏ~É~ÓC^ÐB_Ð_bÐcmÐbnÐnoÐpÐpà-¨c¨U°"°Z°LÀÀJÀ<ÈqÐQÐQà—>—>Ø-¨c¨U°"°Z°LÀÄ;ÈtÏ~É~ÓC^ÐB_Ð_`ÐaÐaà-¨c¨U°"°Z°LÀÐBÐBØÐ0Ó0Ü˜*¤e×,Ñ,à#-Ñ �™&ô $5°^Ó#PÑ �˜&Ø�~�~Ø)¨#¨¨d°4°&¸¸2¸$¸bÀÀÈÌ[ÐY]×YgÑYgÓMhÐLiÐijÐkÐkà)¨#¨¨d°4°&¸¸2¸$¸bÀÀÈÐLÐLØÐ3Ó3ØÑ(Ð(Ð(Ü! )Ñ,ˆFØœEŸJ™JÓ&Ü%¤e§k¡kÑ2�Ø×)Ñ)¨)Ó4ˆEØ×)Ñ)¬%¯+©+Ó6ˆEØˆGØˆIØÑ Ø+Ñ7Ð7Ð7ØœsÐ#7Ó8×>Ñ>Ó@ÐAÐB�Ø   ˜L�	Ø�~�~à%Ð& m°F°8¸2¸e¸WÀBÀuÀgÈgÈYð WØ�u˜B˜z˜l¨9¨+°R¼ÀDÇNÁNÓ8SÐ7TÐTUðWðð
 )Ð)¨°v°h¸bÀÀÀrÈ%ÈÐQXÐPYÐY[Ð\_Ð[`Ð`bÐcmÐbnÐoxÐnyÐyzÐ{Ð{Ø˜uÓ$Ü˜*¤n×5Ñ5Ø!×'Ñ'¬5¯:©:Ó5Ð5Ð5Ü 4´Q·X±X×5EÑ5EÈ
À}Ó U‘�Ø�~�~Ø+¨C¨5°°:°,¸bÄÈTÏ^É^ÓA\Ð@]Ð]^Ð_Ð_à˜˜c * Ð.Ð.ä%Ð%rW   c           	      ó   >• [        U[        5      (       d   eUR                  c   eUR                  (       d<  [        U[        5      (       a  UR                  (       a  SU S3n[        T	U ]  XX45      $ UnUnU(       a!  U R                  UR                  5       SU S3nU(       a!  U R                  UR                  5       SU S3nU(       a!  U(       a  SU SU SU SU S3	nU SU SU 3nO-U(       a  U SU 3nU SU 3nOU(       d   eU SU 3nU SU 3nU R                  UR                  5       SU S3nU(       a;  UR                  (       d!  U R                  UR                  5       SU S3nSU SU S3nU R                  (       aS  U R                  UR                  5       SU R                  UR                  5       S	U S
[        U R                  5       S3nSU S3nU R                   SU SU S3$ )Nr¡  z).all_masked()r–   rT  z) & (r¼   z) | ~(z::set(z::from(1), (r    z, "index out of bounds: z"))r¢   rG   rˆ   r¦   r?  r¢  r  rC  r$  rE   r?  )
rA  r§   r›  rœ  r   Úlower_scalarÚupper_scalarr¹  Ú
cond_printrD  s
            €rX   r¢  ÚCppVecKernel.indirect_assert€  s&  ø€ Ü˜#œ~×.Ñ.Ð.Ð.Ø�y‰yÑ$Ð$Ð$Ø�z�zÜ˜$¤×/Ñ/°D·K·KØ˜4˜& Ð/�Ü‘7Ñ*¨3°uÓCÐCØˆØˆÞØ×)Ñ)¨#¯)©)Ó4Ð5°Q°u°g¸QÐ?ˆEÞØ×)Ñ)¨#¯)©)Ó4Ð5°Q°u°g¸QÐ?ˆEÞ–UØ�u�g˜T #  e¨C¨5°°E°7¸!Ð<ˆDØ(˜>¨¨c¨U°#°l°^ÐD‰JÞØ�W˜D  Ð&ˆDØ(˜>¨¨c¨UÐ3‰JæˆL�5Ø�U˜#˜e˜WÐ%ˆDØ˜5  L >Ð2ˆJØ×%Ñ% c§i¡iÓ0Ð1°°4°&¸Ð:ˆÞØ—;—;Ø×-Ñ-¨c¯i©iÓ8Ð9¸¸4¸&ÀÐB�à�t�f˜F 4 &¨Ð*ˆDØ�>�>à×&Ñ& s§y¡yÓ1Ð2°&¸×9LÑ9LÈSÏYÉYÓ9WÐ8Xð YØ�V˜3œ{¨4¯>©>Ó:Ð;¸1ð>ð ð �4�&˜Ð'ˆØ×&Ñ&Ð' q¨¨Ð.FÀzÀlÐRTÐUÐUrW   c                 óF  >• [        U[        5      (       d   eUR                  (       d  [        T	U ]  XU5      $ [
        U   nU R                  U5      n[
        U   nU R                  U5      nSU S3nU[        R                  :w  a3  U[        R                  :X  a  U R                  U5       SU SU SU S3nU$ U[        R                  :X  a!  U[        R                  :w  a  U SU SU S3nU$ X2:w  a,  XWs=:X  a  S:X  a  O  OS	U SU S3nU$ S	U SU SU SU SU S3nU$ )
Nr¡  r–   z::from<r.  r½  z.to<r‚   r5   rß  )
r¢   rG   r¦   r?  r°  rH   r%  r„   rl   rC  )
rA  rQ  rˆ   rª   Úsrc_cpp_typerå  Údst_cpp_typeÚdst_num_vectorsr¹  rD  s
            €rX   r°  ÚCppVecKernel.get_to_dtype_expr¥  sY  ø€ Ü˜#œ~×.Ñ.Ð.Ð.Ø�z�zÜ‘7Ñ,¨S¸ÓCÐCÜ# IÑ.ˆØ×/Ñ/°	Ó:ˆÜ# EÑ*ˆØ×/Ñ/°Ó6ˆØ�3�%�qˆzˆØœŸ
™
Ó" u´·
±
Ó':Ø×)Ñ)¨)Ó4Ð5°W¸\¸NÈ!ÈOÐK\Ð\^Ð_bÐ^cÐcdÐeˆDð ˆð œ%Ÿ*™*Ó$¨´%·*±*Ó)<Ø�U˜$˜|˜n¨A¨oÐ->¸cÐBˆDð ˆð ÓØÕ6°QÖ6Ø*¨<¨.¸¸3¸%¸qÐA�ð ˆð +¨<¨.¸¸/Ð9JÈ!ÈLÈ>ÐYZÐ[jÐZkÐkmÐnqÐmrÐrsÐt�ØˆrW   )rW  rm  r$  rÓ  r  rl  r  )NNFre  )+r  r  r  r  r   r  r@  rõ   rý  r  r  r„   rˆ   rn   r%  ro   rÒ  rp   r  rC  rG   rê  r   rŽ  r=   r	   rl   r  r�  r·  r¶  rñ  rô  r  r  rÉ  rÈ  r…   rÊ  r¢  r°  r!  ry  rz  s   @rX   r  r  U
  s“  ø† Ø€Ið ÷Cð"4¨5¯:©:ð 4ÀÇÁô 4ð e§k¡kð °cô ðR¨%¯+©+ð R¸%ô Rð
P 5§;¡;ð P°3ô Pð 38·+±+ñ H E§K¡Kð HÀ#õ HðN >ð N¸%¿+¹+ð NÈ#ô Nð /3ñ#àð#ð �z‰zð#ð �{‰{ð	#ð
 ˜NÑ+õ#ðT ,0Ø<@Ø ñMà�c‰]ðMð �z‰zðMð �{‰{ð	Mð
 ˜Ñ(ðMð ˜e C¨Ð$7Ñ8Ñ9ðMð ðMð 
�.Ñ	!õMð^˜ð  U§Z¡Z÷ ð4 !ñ*à�S˜.Ð(Ñ)ð*ð ð*ð �z‰zð	*ð
 �{‰{ð*ð õ*ôX"<òHròh%PðN Nð °~ô ð$
˜Nð 
°E·L±Lð 
À^ô 
ò òDð* Ø(,Ø/3Ø+0¯=©=ñf&ð ˜Ÿ™Ñ%ðf&ð ' t™nðf&ð ˜EŸK™KÑ(õf&÷P#V÷Jó rW   r  c                   óØ   ^ • \ rS rSrSr\r  SU 4S jjrS rS r	 SS jr
S\S\R                  4U 4S	 jjrSU 4S
 jjrS rU 4S jrS\R                  S\R                  4S jrSrU =r$ )rB  iº  a  
A vector kernel that handles the 2d tiles with the tile size defined in `tiling_factor` on
the inner-most loop level and one of the outer loop level (`outer_tiling_idx`). When the data
tile is accessed in a contiguous way from the outer loop axis, a transposition is applied on the
tile to make the access contiguous from the inner-most loop axis. Then, the same vectorization
logic from its parent `CppVecKernel` is leveraged for load/store/compute. The transposed tile load
and store are generated into kernel.preloads and kernel.poststores buffers.

The loop structure looks like below:
for ...
  for i_outer ...
    for ...
      for inner_most ...
        // generated by CppTile2DKernel
        float tmp0[16*16]; at::vec::transpose_mxn<...>(tmp0, in_ptr0 + ..., ...); // into kernel.preloads
        float tmp1[16*16]; // into kernel.preloads
        for i_inner ... { // the kernel inner loop
          vectorized loads/compute/stores (e.g., load tmp0, store tmp1) // into kernel.loads/compute/stores
        }
        at::vec::transpose_mxn(out_ptr0 + ..., tmp1, ...) // into kernel.poststores
      for inner_most ... (tail)
        // generated by CppVecKernel
        ...
  for i_outer ... (tail)
    for ...
      for ...
        // generated by CppKernel
        ...
c                 ó¤   >• [         TU ]  UUUUS   U5        X@l        XPl        X`l        U(       a  UOUU l        U(       a  UOUU l        SU l        g )Nr5   T)r?  r@  Útiling_indicesÚinner_tail_sizeÚouter_tail_sizeÚinner_num_elemsÚouter_num_elemsÚinner_is_tiling_idx)rA  r›  ra  rÓ  r  r  r  rD  s          €rX   r@  ÚCppTile2DKernel.__init__Û  sZ   ø€ ô 	‰ÑØØØØ˜1ÑØô	
ð -ÔØ.ÔØ.ÔÞ2A™À}ˆÔÞ2A™À}ˆÔØ#'ˆÕ rW   c                 óL   • [        U R                  U R                      S35      $ )NrZ  )r-   r  Ú	outer_idxrG  s    rX   Úinner_itervarÚCppTile2DKernel.inner_itervarò  s"   € Ü! T§]¡]°4·>±>Ñ%BÐ$CÀ6Ð"JÓKÐKrW   c                 ó–  • U R                   U R                     nU R                   U R                     n[        XU R                  5      n[        XU R                  5      nU R
                  S L =(       aW    US:H  =(       aK    UR                  U5      =(       a3    UR                  U5      (       + =(       a    UR                  U5      (       + $ ró   )r  r  r  r  rÓ  r~  rô   )rA  r’   Ú	outer_varÚ	inner_varÚouter_strideÚinner_strides         rX   Úneed_vec_transposeÚ"CppTile2DKernel.need_vec_transposeõ  s©   € Ø—M‘M $§.¡.Ñ1ˆ	Ø—M‘M $§/¡/Ñ2ˆ	Ü*¨5¸T×=OÑ=OÓPˆÜ*¨5¸T×=OÑ=OÓPˆà�O‰O˜tÐ#÷ 0Ø Ñ!÷0à—	‘	˜)Ó$÷0ð !×$Ñ$ YÓ/Ô/÷0ð !×$Ñ$ YÓ/Ô/ð	
rW   c                 óÊ  • [         R                  R                  U5      nU R                  nU S[	        U5       3nSn	[	        [        X0R                  U R                     U R                  5      5       n
[	        U R                  5       nU(       a  X˜p˜XºpºSnU R                  U-  (       a  U R                  U R                  píOU R                  U R                  píU(       a  US:X  a  SOSn[        U[        R                  5      (       a  UR                  (       a0  [        U[        R                  5      (       aF  UR                  (       d5  S[         U    SU S	U S
U
 S
U	 S
U S
[	        U5       S
[	        U5       S3nO4S[         U    S[	        U5       S[	        U5       SU S	U S
U
 S
U	 S
U S3nU(       a  U R"                  R%                  5       nOcU R"                  R'                  U5      (       d&  U R"                  R)                  U R*                  USS9nOSnU R"                  R-                  U5      nU(       a>  [         U   nSU SU S3nU SU SU SU SU S3
nU R*                  R/                  U5        UR1                  S[3        U5      5      nU(       a'  U R4                  R/                  [7        UU5      5        U$ U R*                  R/                  U5        U$ )Nré   Ú__place_holder__Tr²  ÚtrueÚfalseztranspose_mxn<r.  r½  rœ   rÖ   Frè  zalignas(std::max(std::size_t(z), alignof(z)))r˜   r©  r\   rØ   )r4   r¬  r­  rÓ  rE   r  r  r  rm  r  r	  r
  r¢   rõ   rý  r  rH   r±  rZ  Úcontainsr²  r_  ÚgetrÂ   r  rp   r`  r;   )rA  rÇ   r§   r’   Úis_storeÚ
store_moderˆ   ÚfactorrQ  rT  Úld_srcÚld_dstÚneed_defineÚMÚNr²  Úload_or_storeÚtile_varÚ	cpp_dtypeÚalignasÚdefine_lines                        rX   Úgen_transposed_tile_load_storeÚ.CppTile2DKernel.gen_transposed_tile_load_store  sÁ  € ô —‘×!Ñ! $Ó'ˆØ×#Ñ#ˆØ��Sœ UÓ+Ð,Ð-ˆØ ˆÜÔ 3°E¿=¹=ÈÏÉÑ;YÐ[_×[mÑ[mÓ nÓoÐpˆÜ §¡Ó/Ð0ˆÞØ�Ø#�FàˆØ×#Ñ# h×.Ø×'Ñ'¨×)=Ñ)=‰qð ×$Ñ$Ø×$Ñ$ð ö !)¨j¸LÓ.H‘VÐPWˆ
Ü�qœ%Ÿ*™*×%Ñ%¨a¯k¯kÜ�qœ%Ÿ*™*×%Ñ%¨a¯k¯kð !¤¨eÑ!4Ð 5°Q°z°lð CØ�5˜˜6˜( " S E¨¨F¨8°2´kÀ!³nÐ5EÀRÌÐTUËÐGWÐWYð[ñ ð !¤¨eÑ!4Ð 5°Q´{À1³~Ð6FÀaÌÐTUËÐGWÐWXÐYcÐXdð eØ�5˜˜6˜( " S E¨¨F¨8°2ð7ð ö Ø—x‘x—‘Ó(‰HØ—‘×"Ñ" =×1Ñ1Ø—x‘x×(Ñ(¨¯©¸ÈUÐ(ÐS‰HàˆKØ—x‘x—|‘| MÓ2ˆHæÜ$ UÑ+ˆIð 6°f°X¸[ÈÈÐSVÐWˆGØ$˜I Q y k°°8°*¸A¸f¸XÀQÀvÀhÈbÐQˆKØ�M‰M×#Ñ# KÔ0à%×-Ñ-Ð.@Ä#ÀhÃ-ÓPˆÞØ�O‰O×%Ñ%¤l°4¸Ó&GÔHð ˆð �M‰M×#Ñ# MÔ2àˆrW   rÇ   r’   c                 óX  >• U R                   R                  U5      nU R                  U5      nU R                  5       nU R	                  U5      (       a¸  U R                  XUSS9nU S[        X@R                  -  5       3n[        R                  R                  U5      nU R                  USU5      nU R                  R                  U R                  X‡S9n	U	R                  SXU40 5        [!        U	["        5      (       d   eSU	l        U	$ U R'                  U5      n
[(        TU ]U  X5      $ )NF)r  ré   r   r  r�  T)r›  r«  r˜  r  r  r+  rE   rm  r4   r¬  r­  rŽ  r±  r²  rx  r´  r¢   rG   r¦   rC  r?  r�  )rA  rÇ   r’   r§   rq  r'  r�  rˆ   rÈ   rº  rú   rD  s              €rX   r�  ÚCppTile2DKernel.load>  s  ø€ Ø�i‰i�o‰o˜dÓ#ˆØ×$Ñ$ UÓ+ˆà×"Ñ"Ó$ˆØ×"Ñ" 5×)Ñ)Ø×:Ñ:Ø˜5¨5ð ;ð ˆHð "˜
 #¤k°%¿.¹.Ñ2HÓ&IÐ%JÐKˆGÜ—G‘G×%Ñ% dÓ+ˆEØ×*Ñ*¨7°A°uÓ=ˆDØ—X‘X×&Ñ& t§z¡z°4Ð&ÐEˆFØ×!Ñ! &¨4°uÐ*=¸rÔBÜ˜f¤n×5Ñ5Ð5Ð5Ø ˆFŒMØˆMà×/Ñ/°Ó6ˆIÜ‘7‘< Ó0Ð0rW   c                 óZ  >• SU;   d   e[        U[        5      (       d   U5       eUR                  (       d  U R                  U5      nU R                  R                  U5      nU R                  5       nU R                  U5      nU R                  U5      (       añ  U R                  XUSUS9nU S[        X`R                  -  5       3nU R                  (       df  [        R                  R                  U5      [         ["        R$                  ["        R&                  ["        R(                  ["        R*                  /-   ;   a  U SU S[        U R                  5       S3n	OU SU S3n	U R,                  R/                  [1        X5      5        g U R3                  U5      n
[4        TU ]m  XX45        g )Nr°  T)r  r  ré   rµ  rœ   rÖ   )r¢   rG   r¦   r  r›  r´  r  r˜  r  r+  rE   rm  r$  r4   r¬  r­  rƒ   r„   rø  rù  rú  rû  rq  rÂ   r;   rC  r?  r¶  )rA  rÇ   r’   rï   rµ  r§   rq  r'  ÚstorebufrÈ   rú   rD  s              €rX   r¶  ÚCppTile2DKernel.storeT  sr  ø€ Ø˜‹}Ðˆ}Ü˜%¤×0Ñ0Ð7°%Ó7Ð0Ø�|�|à—N‘N 5Ó)ˆEà�i‰i×Ñ˜tÓ$ˆà×"Ñ"Ó$ˆØ×$Ñ$ UÓ+ˆØ×"Ñ" 5×)Ñ)Ø×:Ñ:Ø˜5¨4¸Dð ;ð ˆHð #˜ 3¤{°5¿>¹>Ñ3IÓ'JÐ&KÐLˆHØ�~�~¤§¡×!2Ñ!2°4Ó!8¼MÜ—‘Ü—
‘
Ü×#Ñ#Ü×!Ñ!ð	Mñ =ó "ð  ˜ ¨ z°´KÀÇÁÓ4OÐ3PÐPRÐS‘à˜ ¨ z°Ð4�Ø�K‰K×!Ñ!¤,¨tÓ":Õ;à×/Ñ/°Ó6ˆIÜ‰G‰M˜$¨5Õ7rW   c                 ó  • U R                  5       nU R                  (       a2  UR                  SU SU S[        U R                  5       SU S3	5        g UR                  SU SU S[        U R
                  5       SU S3	5        g )Nr¥  r¦  r¼   r½   r¾   )r  r  rÂ   rE   r
  r	  )rA  r³   rq  s      rX   rv  Ú#CppTile2DKernel.codegen_inner_loopss  s�   € Ø×"Ñ"Ó$ˆØ×#×#Ø�N‰NØ˜U˜G 6¨%¨°´KÀ×@TÑ@TÓ4UÐ3VÐVXÐY^ÐX_Ð_bÐcõð �N‰NØ˜U˜G 6¨%¨°´KÀ×@TÑ@TÓ4UÐ3VÐVXÐY^ÐX_Ð_bÐcõrW   c                 óž  >• [         TU ]  X5      nU R                  S   U R                  :  a  U R                  O[	        U R                  5      u  U l        U l        U R                  U R                  S   :X  a+  U R                  U l        U R                  U l
        SU l        U$ U R                  U l        U R                  U l
        SU l        U$ )Nr5   r   FT)r?  rú  r  rO  Úreversedr  r  r  r$  r
  rm  r  r  r	  )rA  ÚgroupÚreduction_groupr  rD  s       €rX   rú  ÚCppTile2DKernel.set_ranges~  s¾   ø€ Ü‰wÑ! %Ó9ˆð ×"Ñ" 1Ñ%¨×(<Ñ(<Ó<ð ×Òä˜$×-Ñ-Ó.ñ 	(ˆŒ˜œð
 �?‰?˜d×1Ñ1°!Ñ4Ó4Ø!×1Ñ1ˆDŒNØ!×1Ñ1ˆDŒNØ',ˆDÔ$ð
 ˆð "×1Ñ1ˆDŒNØ!×1Ñ1ˆDŒNØ'+ˆDÔ$ØˆrW   r¸   c                 óT   • U R                  UU R                  U R                  5       S9$ )Nr¤  )r„  r  r  r‡  s     rX   rC  Ú"CppTile2DKernel.transform_indexing�  s0   € Ø×+Ñ+ØØŸ™Ø×%Ñ%Ó'ð ,ð 
ð 	
rW   )
r  r	  r  rm  r  r
  r  r$  r  r  rf  r  )r  r  r  r  r  r@  r  r@  r  r  r+  rp   rõ   rý  r�  r¶  rv  rú  rC  r!  ry  rz  s   @rX   rB  rB  º  sx   ø† ñð< #€Ið Ø÷(ò.Lò
ð 6:ô:ðx1˜ð 1 U§Z¡Z÷ 1÷,8ò>	õð$
¨¯
©
ð 
°u·z±z÷ 
ò 
rW   rB  Ú_bodyc                 ó²  • U R                   /[        U R                  R                  5       5      -   nSnSnU GH  nUR                  R
                   Hû  nUR                  S:X  d  UR                  S;   a  M%  UR                  S;  a  Sn[        US5      (       a±  UR                  (       a   [        R                  UR                  ;   d   eUR                  [        R                     nUR                  (       a  UR                  [        ;  a  SnM¿  Ub)  X&R                  :w  a  [        R                  " S5        Mé  Më  UR                  nMù  SnMý     GM     X#4$ )	zÎ
Returns the low precision data type (torch.float16/torch.bfloat16) contained in the nodes
and if all the nodes can codegen with this data type without converting to float.
Otherwise returns None and True.
NFÚplaceholder)Ú	get_indexrš  )r�  r¶  rÂ  rÌ  r´  Tr…  z.bf16 and fp16 are mixed in the scheduler node.)Ú
root_blockr5  Ú	subblocksr,  r¬  ÚnodesÚopÚtargetr¥   r…  rA   r„  rˆ   rƒ   ÚwarningsÚwarn)r;  Ú
sub_blocksÚ_lowp_fp_typeÚ	_use_fp32Ú	sub_blockrC  r€  s          rX   Úget_loop_body_lowp_fprJ  ˜  s   € ð ×"Ñ"Ð#¤d¨5¯?©?×+AÑ+AÓ+CÓ&DÑD€Jà+/€MØ€IÜˆ	Ø—_‘_×*Ô*ˆEØ�x‰x˜=Ó(¨E¯L©Lð =ó -ñ ð �|‰|ð $ó ð !�	ä�u˜f×%Ñ%¨%¯*¯*Ü*×.Ñ.°%·*±*Ó<Ð<Ð<Ø/4¯z©zÔ:M×:QÑ:QÑ/R�Ø—}—}¨¯©¼]Ó(JØ $’IØ"Ñ.Ø$¯©Ó5Ü ŸšÐ&VÖWñ 6ð %,§M¡M’Mà ’	ô9 +ñ  ð> Ð#Ð#rW   c                   óB   • \ rS rSrSrS\\\   \\   4   4S jrS r	Sr
g)ÚTilingSelectiÄ  zŒ
Implement the heuristic to select the tiling factors and tiling indices.
In the future, we can implement advanced heuristic in a subclass.
r¸   c           	      óL	  ^#• [        U5      n[        U5      nU(       d   e[        S U 5       5      (       a  / / 4$ [        R                  n[        US   5      S   m#T#(       a  [        U#4S jUSS   5       5      (       a  T#n[        R                  " 5       R                  US9nU R                  XU5      nU(       GaÞ  [        US S9u  p‰[        U5      [        U	5      -   n
[        R                  R                  (       GaŽ  S nS	 nS
 n[!        [#        U
5      5       Vs/ s H  n[%        [&        R(                  U5      PM     nn[#        U5      nUS U UUS  nn0 n0 nU GH�  nUR*                  /[-        UR.                  R1                  5       5      -   nU GHS  nUR2                  R4                   GH4  nUR6                  S;   a’  UR6                  S:X  a  SOSnUR8                  R;                  UU45      UR<                  U   R<                  S      nU" X÷5      (       a7  U" UXöU5      nUR6                  S:X  a  Uc  OUS;  a  U" UR6                  U5        [?        UR6                  [@        5      (       d  MÇ  UR6                  RC                  S5      (       a  Mé  UR6                  S;   a  Mû  UR6                  U;  a  SUUR6                  '   GM  UUR6                  ==   S-  ss'   GM7     GMV     GM’     [E        UR1                  5       5      n[E        UR1                  5       5      nSnSnUU:¼  d  US:”  a  UU-  U:¼  a  / / 4$ U	(       dD  U(       a=  [#        U5      S:X  a.  [G        X‡S      /5      (       d  X‡S      US-  :  a
  US:  a  / / 4$ U[H        ;   aÚ  [        R                  " 5       R                  US9n U H²  n!U!S:  a  U![#        U
5      -   n!U!S:  d  U![#        U
5      :¼  a  M.  [G        U
5      (       ad  [J        R2                  RL                  RO                  U
U!   SS9n"U"U :  a1  [J        R2                  RL                  RQ                  U"U 5        U S-  n  OM¢  U
U!   U :  d  M­  U S-  n  O   [#        U5      S:X  a  U/U4$ [#        U5      S:X  a  Xf/U4$ / / 4$ s  snf )Nc              3   ó2   #   • U  H  o[         ;  v •  M     g 7fr  )rw   )r*  rˆ   s     rX   r,  Ú-TilingSelect.select_tiling.<locals>.<genexpr>Ó  s   é € ÐHºZ°EÔ/Ö/ºZùs   ‚r   c              3   óF   >#   • U  H  n[        U5      S    T:H  v •  M     g7f)r   N)rJ  )r*  Ú	loop_bodyÚ_lowp_fp_dtypes     €rX   r,  rO  ×  s'   øé € ð "
â,�	ô # 9Ó-¨aÑ0°NÖBÚ,ùó   ƒ!r5   r  c                 ó   • [        U S   5      $ ró   ©rÓ   ©Úsizess    rX   r½  Ú,TilingSelect.select_tiling.<locals>.<lambda>ä  s   € ´#°e¸A±h´-rW   ©r„  c                 óR   • XS      n[        XU5      nUR                  (       a  U$ S $ rm  )r  r  )r’   r  rÓ  r  rŠ  r  s         rX   Ú_try_get_strideÚ3TilingSelect.select_tiling.<locals>._try_get_strideê  s0   € ð '°aÑ'8Ñ9�GÜ0°ÀÓO�FØ%+×%5×%5˜6Ð?¸4Ð?rW   c                 ó0   • X;  a  SX'   g X==   S-  ss'   g ró   rV   )Ú	node_nameÚnon_contig_indexing_op_counters     rX   Ú_update_negative_op_countÚ=TilingSelect.select_tiling.<locals>._update_negative_op_countô  s!   € ð !ÓFØDEÐ6ÒAà6ÓAÀQÑFÔArW   c                 ó¬   • [        U5      S:H  =(       a@    [        U 5      S:„  =(       a+    US   S:¼  a  US   OUS   [        U 5      -   [        U 5      :  $ ©Nr5   r   rU  )r  r  s     rX   Ú_is_valid_indicesÚ5TilingSelect.select_tiling.<locals>._is_valid_indicesü  sb   € ô
 ˜NÓ+¨qÑ0÷ (Ü ›M¨AÑ-÷(ð  .¨aÑ0°AÓ5ð +¨1Ò-à!/°Ñ!2´S¸³]Ñ!Bä˜h›-ñ(ð	rW   )rš  r�  r¶  rš  r   ©r   r5   Úmasked_subblock)r2   r´  r“  r>  g¸…ëQ¸¾?é#   r•  é
   rþ  ))rC   rB   rg   r„   ro   rJ  r4  r   rk  Ú	nelementsÚ_select_tiling_indicesr`   r£   r   rð  Úenable_tiling_heuristicsr^  rÓ   r.   r   r÷  r?  r5  r@  r,  r¬  rA  rC  r¡  Úindexing_from_argsr›  r¢   rp   Ú
startswithrd   r(   rƒ   r4   rÉ  rÊ  rÌ  )$rA  Úfn_listÚvar_sizes_listÚloop_bodiesÚ
all_dtypesrˆ   rÓ  r  r6  r7  rN  r[  r`  rd  rp  r  rO  r  Úreduction_varsÚ
op_counterr_  r;  rF  rI  rC  Úarg_idxr’   r  Úop_numÚnon_contig_indexing_op_numÚratio_thresholdÚquantity_thresholdÚfactor_lowpÚtiling_indiceÚ
call_rangerR  s$                                      @rX   Úselect_tilingÚTilingSelect.select_tilingÊ  s²  ø€ ô % WÓ-ˆÜ/°Ó<ˆ
ÞÐˆzÜÑH¹ZÓH×HÑHØ�r�6ˆMÜ—‘ˆÜ.¨{¸1©~Ó>¸qÑAˆÞœcô "
à(¨¨™_ó"
÷ 
ñ 
ð #ˆEä#×0Ò0Ó2×<Ñ<À5Ð<ÐIˆØ×4Ñ4Ø ]ó
ˆ÷ Ü%(ØÑ$?ñ&Ñ"ˆEô   ›,¬¨Ó)?Ñ?ˆKä�z‰z×2×2Ð2ò@òGòô" #¤3 {Ó#3Ô4óâ4˜ô 3´4·;±;ÀÖBÙ4ð ð ô #& e£*�àÐ-˜oÐ.Ø˜_Ð-Ð.ð %�ð .0�
ð BDÐ.Ü(�EØ"'×"2Ñ"2Ð!3´d¸5¿?¹?×;QÑ;QÓ;SÓ6TÑ!T�JÜ%/˜	Ø%.§_¡_×%:Õ%:˜EØ$Ÿ|™|Ð/NÓNà/4¯|©|¸|Ó/K©!ÐQR Ø(1¯©×(IÑ(IØ%)¨>Ð$:ó)"à"'§*¡*¨WÑ"5×":Ñ":¸1Ñ"=ñ)? ñ $5°X×#NÑ#NÙ-<Ø(-¨xÈó.& Fð
 ,1¯<©<¸<Ó+Gð )/ªà-3¸6Ó-Aá(AØ,1¯L©LÐ:Xô)*ô  *¨%¯,©,¼×<Ó<Ø %§¡× 7Ñ 7Ð8I× JÓ JØ#(§<¡<Ø#Mõ$Nð $)§<¡<°zÓ#AØ?@ J¨u¯|©|Ô$<à$.¨u¯|©|Ó$<ÀÑ$AÖ$<ô7 &;ô &0ñ )ô@ ˜Z×.Ñ.Ó0Ó1�Ü-0Ø2×9Ñ9Ó;ó.Ð*ð #'�Ø%'Ð"Ø-Ð1CÓCØ˜Q“JØ2°VÑ;¸ÓNð
 ˜r˜6�Mö (ÞÜ˜NÓ+¨qÓ0Ü,à!°Ñ"3Ñ4ð÷ñ ð
 ¨QÑ/Ñ0°=À1Ñ3DÓDØ ›ð ˜r˜6�MàœÓ%ô *×6Ò6Ó8×BÑBÈÐBÐO�Û%3�MØ$ qÓ(Ø(5¼¸KÓ8HÑ(H˜Ø$ qÓ(¨M¼SÀÓ=MÓ,MÙ Ü'¨×4Ñ4Ü%&§W¡W×%5Ñ%5×%?Ñ%?Ø'¨Ñ6Àð &@ð &˜
ð &¨Ó3ÜŸG™G×,Ñ,×5Ñ5°jÀ+ÔNØ,7¸1Ñ,<˜MÙ!ñ 4ð % ]Ñ3°kÕAØ(3°qÑ(8˜Ùñ &4ô" �>Ó" aÓ'Ø%�¨Ð6Ð6Ü�>Ó" aÓ'Ø%Ð5°~ÐEÐEØ�2ˆvˆùòQs   Ä"$R!c           	      óö  • / n[        X5       Hd  u  pV[        R                  " U/UQ76 nU[        R                  " UR
                  UR                  5       Vs/ s H  oˆR                  PM     sn-  nMf     [        [           " 5       n	/ n
[        [           " 5       n[        [           " 5       nU GH  nUR                   GH  n[        R                  " SUR                  5      (       d  M,  [        XÞU5      nUS:X  a  M@  US:X  aP  U	R                  [        UR                  SS  5      5        U
R!                  [        UR                  SS  5      5        M–  [#        S UR                   5       5      (       a)  UR                  [        UR                  SS  5      5        Mà  UR                  [        UR                  SS  5      5        GM
     GM     X›-
  U-
  n[%        US S9u  nn['        U5      ['        U5      -   n['        U	5      S:X  a  US-
  /$ U(       a  [)        U5      SS  $ X›-  U-
  n[)        U	5      n['        U5      S:X  a  US   U;   a  US   US-
  :X  a  U$ [)        UU
R*                  S9SS  $ s  snf )	Nz^d\d+$r   r5   c              3   óV   #   • U  H  n[        U[        R                  5      v •  M!     g 7fr  )r   r   ÚSIZE©r*  r�  s     rX   r,  Ú6TilingSelect._select_tiling_indices.<locals>.<genexpr>�  s    é € ÐSÒ?R¸!œ¨¬4¯9©9×5Ð5Ò?Rùó   ‚')c                 ó   • [        U S   5      $ ró   rU  rV  s    rX   r½  Ú5TilingSelect._select_tiling_indices.<locals>.<lambda>’  ó   € ÄsÈ5ÐQRÉ8Ä}rW   rY  rc  r   )r]  r
   Úextract_read_writesÚ	itertoolsÚchainÚreadsÚwritesr’   r   rn   r‘  rÉ   rê   rÇ   r  r¦  r  r4  r`   rÓ   ÚsortedÚcount)rA  ro  rp  rÓ  Ú	all_indexÚfnÚ	var_sizesÚrwÚdepÚcontig_varsÚcontig_vars_listÚnon_contig_stride_constÚnon_contig_stride_otherr’   r§   r  Úcontig_onlyr6  r7  Únum_itervarsÚcontig_and_const_strideÚcontig_vars_sorteds                         rX   rk  Ú#TilingSelect._select_tiling_indicesu  s?  € ð ˆ	Ü  Ö9‰MˆBÜ×1Ò1°"ÐA°yÒAˆBØ¬y¯ª¸r¿x¹xÈÏÉÔ/SÓTÒ/S¨Ÿ)œ)Ñ/SÑTÑTŠIñ :ô !¤’oÓ'ˆØÐÜ",¬S¢/Ó"3ÐÜ",¬S¢/Ó"3ÐÜˆEØ×)Õ)�Ü—y’y ¨C¯H©H×5Ñ5ÙÜ,¨U¸ÓG�Ø˜Q“;ÙØ˜q“[Ø—O‘O¤C¨¯©°°¨Ó$5Ô6Ø$×+Ñ+¬C°·±¸¸°Ó,=Ö>ÜÑS¸v×?RÒ?RÓS×SÑSØ+×/Ñ/´°C·H±H¸Q¸R°LÓ0AÖBà+×/Ñ/´°C·H±H¸Q¸R°LÓ0A×Bô *ñ ð "Ñ;Ð>UÑUˆÜ!$ ^Ñ9TÑ!UÑˆˆÜ˜5“z¤C¨Ó$8Ñ8ˆÜˆ{Ó˜qÓ à  1Ñ$Ð%Ð%ÞÜ˜+Ó& r sÐ+Ð+àÑ1Ø#ñ#$Ðô $ KÓ0ÐäÐ"Ó# qÓ(Ø" 2Ñ&Ð*AÓAØ" 2Ñ&¨,¸Ñ*:Ó:à%Ð%ÜÐ(Ð.>×.DÑ.DÑEÀbÀcÐJÐJùòK Us   ÁI6rV   N)r  r  r  r  r  r£   r5  rn   r}  rk  r!  rV   rW   rX   rL  rL  Ä  s1   † ñð
ið 
ˆt�C‰y˜$˜s™)Ð#Ñ	$ô	iõV.KrW   rL  c                   óê   ^ • \ rS rSr% \r\\   \S'   \r	\\   \S'   \
r\\
   \S'   U 4S jrS rS\4S jrS	\4S
 jrS rS rS rS\\   4S jrS rS rSS\\   4S jjrS\S\S   4S jrSrU =r$ )r  i¦  Ú
kernel_clsÚvec_kernel_clsÚtile2d_kernel_clsc                 óÌ   >• [         TU ]  UR                  UR                  R                  5        Xl        S U l        S U l        [        R                  " 5       U l
        / U l        g r  )r?  r@  r›  Úwsra  rn  r_  rN  r   rk  Úpicked_vec_isaÚkernels©rA  rn  rD  s     €rX   r@  ÚCppKernelProxy.__init__±  sO   ø€ Ü‰Ñ˜×*Ñ*¨L¯O©O×,GÑ,GÔHØ(ÔØˆŒØˆÔÜ2=×2JÒ2JÓ2LˆÔØ(*ˆ�rW   c                 óp   • U H0  n[        U[        5      (       d   e[        R                  " U5        M2     g r  )r¢   r$   r:   Úpropagate_scheduler_node)rA  rA  rC  s      rX   Údata_type_propagationÚ$CppKernelProxy.data_type_propagation¹  s-   € ÛˆEÜ˜e¤]×3Ñ3Ð3Ð3Ü×8Ò8¸Ö?ò rW   Úscheduler_nodec                 óê   • [        UR                  [        5      (       d  g[        R                  " U5        [        UR                  5      S   S L=(       a    [        UR                  5      S   (       + $ )NTr   r5   )r¢   r;  r   r:   r¨  rJ  )rA  r«  s     rX   Úis_lowp_fp_schedulerÚ#CppKernelProxy.is_lowp_fp_scheduler¿  s_   € Ü˜.×.Ñ.´×9Ñ9Øä×4Ò4°^ÔDä! .×"6Ñ"6Ó7¸Ñ:À$ÐF÷ CÜ)¨.×*>Ñ*>Ó?ÀÑBÔBð	
rW   rQ  c                 óÚ   • S[         R                  R                  4S jnUR                  /[	        UR
                  R                  5       5      -   nU H  nU" UR                  5        M     g )NÚ	sub_graphc                 ó  ^^^^^^^^• S[         R                  R                  S[        [         R                     4S jmS[         R                  R                  S[        [         R                     4S jmS[         R                  R                  S[         R                  4U4S jjmS[         R                  R                  S[         R                  4U4S jjmS[         R                  R                  S[         R                  4UU4S jjn[        U R                  5      n/ mU GHl  nUR                  S	;   a³  T" U5      =m[        ;   a¡  [        UU4S
 jUR                   5       5      (       a  MM  UR                  S   nU R                  U5         U R                  SXC[         R                  4S9mUR                  TU4S j5        [         =R"                  S-  sl        S S S 5        MÇ  UR                  S:X  a¬  T" U5      =m[        ;   aš  UR                  u  pEpgnU" UT5      (       a  GM
  [$        R&                  R)                  U5      mU R+                  U5         U R                  SXGT4S9mUR-                  UT5        [         =R"                  S-  sl        S S S 5        GMƒ  UR                  S:X  aš  UR                  u  nmnn	n
U[        ;   a{  T[         R                  [         R.                  [         R0                  [         R2                  4;   d   eUT[        ;   a  [         R                  OT[         R                  U	U
4Ul        GM*  GM-  UR                  S:X  ah  UR                  S   [        ;   aQ  UR                  u  pJm[        UU4S jUR                   5       5      (       a  GM‹  XJ[         R                  4Ul        GM¥  UR                  S:X  ay  UR                  S   [        ;   ab  UR                  u  pKm[        UU4S jUR                   5       5      (       a  GM  TR5                  U5        XK[         R                  4Ul        GM.  UR                  S:X  d  GMA  UR                  u  pGmnU[        ;   ad  U" Xx5      (       dW  U R+                  U5         U R                  SXGU4S9mUR-                  UT5        [         =R"                  S-  sl        S S S 5        T[        ;   d  GMÌ  [        UU4S jUR                   5       5      (       a  GMô  UR                  S   nU R                  U5         U R                  SXC[         R                  4S9mUR                  TU4S j5        [         =R"                  S-  sl        S S S 5        GMo     S[         R                  R6                  4U4S jjnU" U 5        g ! , (       d  f       GM¬  = f! , (       d  f       GM¿  = f! , (       d  f       GN= f! , (       d  f       GMä  = f)Nr+  r¸   c                 ób  • U R                   S:X  a,  [        R                  R                  U R                  S   5      $ U R                   S:X  a  U R                  S   $ U R                   S:X  aD  [        U R                  5      S:”  a  U R                  S   $ U R                  R                  SS5      $ g)	z6Get input dtype for nodes that may consumes lowp fp dtr¶  r5   r¾  rc  r¯  r   rª   N)rC  r4   r¬  r­  r›  rÓ   r	  r  ©r+  s    rX   Úget_input_dtypeÚ]CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.get_input_dtypeË  sˆ   € à—;‘; 'Ó)ÜŸ7™7×,Ñ,¨T¯Y©Y°q©\Ó:Ð:Ø—[‘[Ð$6Ó6ØŸ9™9 R™=Ð(Ø—[‘[ JÓ.Ü˜4Ÿ9™9“~¨Ó)Ø#Ÿy™y¨™|Ð+à#Ÿ{™{Ÿ™¨{¸DÓAÐAàrW   c                 ó.  • U R                   S:X  aG  [        U R                  5      S:X  d   e[        R                  R                  U R                  S   5      $ U R                   S;   a  U R                  S   $ U R                   S:X  a  U R                  S   $ g)	z6Get output dtype for nodes that may produce lowp fp dtr�  r   r5   )r¯  r“  rš  rc  r¾  r   N)rC  rÓ   r›  r4   r¬  r­  r³  s    rX   Úget_output_dtypeÚ^CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.get_output_dtypeÙ  sz   € à—;‘; &Ó(Ü˜tŸy™y›>¨QÓ.Ð.Ð.ÜŸ7™7×,Ñ,¨T¯Y©Y°q©\Ó:Ð:Ø—[‘[Ð$JÓJØŸ9™9 R™=Ð(Ø—[‘[Ð$6Ó6ØŸ9™9 Q™<Ð'àrW   rœ  c                 ó2   >• U[         ;   d   eT" U 5      U:H  $ )z]Check if the given node produces output with expected low precision floating point data type.)rƒ   )r+  rœ  r·  s     €rX   Úis_lowp_fp_sourceÚ_CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.is_lowp_fp_sourceå  s!   ø€ àœ]Ó*Ð*Ð*Ù'¨Ó-°Ñ3Ð3rW   c                 óf   >• U[         ;   d   eT" U 5      =n(       a  X!:H  $ U R                  S:X  a  gg)zZCheck if the given node accept input with expected low precision floating point data type.r¯  TF)rƒ   rC  )r+  rœ  Úinput_dtyper´  s      €rX   Úis_lowp_fp_sinkÚ]CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.is_lowp_fp_sinkê  s<   ø€ àœ]Ó*Ð*Ð*Ù"1°$Ó"7Ð7�;Õ7Ø&Ñ,Ð,Ø—[‘[ JÓ.àà rW   c                 óf   >^• T" U T5      =(       a     [        UU4S jU R                   5       5      $ )z…Check if the node is a lowp fp sources which are all directly fed to ops that accepts lowp fp input
thus no need to promote to float
c              3   ó6   >#   • U  H  nT" UT5      v •  M     g 7fr  rV   ©r*  Úuserrœ  r¾  s     €€rX   r,  Ú}CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.is_lowp_fp_source_no_promote.<locals>.<genexpr>ù  s   øé € ð ;Ú:D°$‘O D¨"×-Ð-º*ùó   ƒ©r4  Úusers)r+  rœ  r¾  rº  s    `€€rX   Úis_lowp_fp_source_no_promoteÚjCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.is_lowp_fp_source_no_promoteõ  s1   ù€ ñ )¨¨rÓ2÷ ´sõ ;Ø:>¿*º*ó;ó 8ð rW   )r�  rš  c              3   ó6   >#   • U  H  nT" UT5      v •  M     g 7fr  rV   rÂ  s     €€rX   r,  ÚWCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.<genexpr>  ó   øé € ÐMÂ¸™?¨4°×4Ð4ÂùrÅ  r   r¯  ©r›  c                 ó   >• U TL$ r  rV   ©rp  Úto_type_nodes    €rX   r½  ÚVCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.<lambda>  s
   ø€ °A¸\Ñ4IrW   r5   r¶  rñ  r“  rc  c              3   ó6   >#   • U  H  nT" UT5      v •  M     g 7fr  rV   rÂ  s     €€rX   r,  rË  @  rÌ  rÅ  c              3   ó6   >#   • U  H  nT" UT5      v •  M     g 7fr  rV   rÂ  s     €€rX   r,  rË  F  rÌ  rÅ  r¾  c              3   ó6   >#   • U  H  nT" UT5      v •  M     g 7fr  rV   )r*  rÃ  rˆ   r¾  s     €€rX   r,  rË  o  s   øé € ÐUÊÀ¡°°e× <Ð <ÊùrÅ  c                 ó   >• U TL$ r  rV   rÏ  s    €rX   r½  rÑ  w  s
   ø€ ¸AÀ\Ñ<QrW   r°  c                 óZ   >• S[         R                  R                  4U4S jjnU" U 5        g )Nr°  c                 óˆ  >^• S[         R                  R                  4S jnU R                   Vs/ s H  o"R                  S:X  d  M  UPM     nnU Vs/ s H  o!" U5      (       d  M  X"R
                  0PM      nnU H›  nUR                  5        H„  u  mnTU R                  ;   d  M  [        U4S jU 5       5      (       d!  TT;   d  M:  [        S U 5       5      (       d  MS  TR                  S   nTR                  U5        U R                  T5        M†     M�     U R                  c  U R                  5         g g s  snf s  snf )NÚto_nodec                 ó:   • [        S U R                   5       5      $ )Nc              3   ó>   #   • U  H  oR                   S :H  v •  M     g7f)r¯  N)rC  ©r*  Úusrs     rX   r,  Ú®CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype.<locals>._eliminate_duplicate_to_node.<locals>._used_by_to.<locals>.<genexpr>…  s   é € Ð"UÂ}À§:¡:°Ö#;Â}ùs   ‚rÆ  )rØ  s    rX   Ú_used_by_toÚ›CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype.<locals>._eliminate_duplicate_to_node.<locals>._used_by_to„  s   € Ü"Ñ"UÀwÇ}Â}Ó"UÓUÐUrW   r¯  c              3   ó`   >#   • U  H#  oR                   S    TR                   S    :H  v •  M%     g7f©rc  NrÍ  )r*  rÜ  r+  s     €rX   r,  Ú™CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype.<locals>._eliminate_duplicate_to_node.<locals>.<genexpr>�  s$   øé € Ð#SÊUÀc§H¡H¨R¡L°D·I±I¸b±MÖ$AÊUùs   ƒ+.c              3   óL   #   • U  H  oR                   S    [        ;   v •  M     g7frá  )r›  rƒ   rÛ  s     rX   r,  râ  “  s   é € ð ,&ÚQVÈ#¯©°©¼Ö(EÒQVùrÏ  rc  )r„   ÚfxÚNoderA  rC  rÇ  r  r4  Úall_input_nodesÚreplace_all_uses_withÚ
erase_nodeÚowning_moduleÚlint)	r°  rÞ  r+  Úall_to_nodesÚall_to_nodes_and_usersÚ
node_usersrÇ  Úval_nodeÚto_lowp_fp_legalized_nodess	     `     €rX   Ú_eliminate_duplicate_to_nodeÚ†CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype.<locals>._eliminate_duplicate_to_node}  s-  ù€ ðV¬U¯X©X¯]©]ô Vð *3¯ªó$Ú)8 ¿K¹KÈ:Ñ<UŸ©ð !ð $ñ 8Dó.Ú7C¨tÀ{ÐSW×GXÓ*˜Ÿz™zÓ*±|ð +ð .ó '=˜
Ø+5×+;Ñ+;Ö+=™K˜D %Ø# y§¡Õ6Ü #Ô#SÉUÓ#S× SÑ Sà$(Ð,FÕ$FÜ(+ñ ,&ÙQVó,&÷ )&ó )&ð
 ,0×+?Ñ+?ÀÑ+C Ø $× :Ñ :¸8Ô DØ )× 4Ñ 4°TÖ :ó ,>ñ '=ð, !×.Ñ.Ñ6Ø!Ÿ™Õ(ð 7ùò9$ùò.s   °D:ÁD:ÁD?Á'D?)r„   rä  ÚGraph)r°  rð  rï  s     €rX   Úeliminate_to_dtypeÚ`CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype|  s"   ø€ ð')¼E¿H¹H¿N¹N÷ ')ñR -¨YÕ7rW   )r„   rä  rå  r   rˆ   r5  rA  rC  rƒ   r4  rÇ  r›  Úinserting_afterÚcall_methodro   rç  r   Úcpp_to_dtype_countr4   r¬  r­  Úinserting_beforeÚreplace_input_withrö  r÷  r  r  rò  )r°  rÈ  Úsub_graph_nodesrC  r2   rÇ   rÍ   Ú	value_varrª   r‡   rï   r·  ró  rœ  rˆ   r´  r·  r¾  rº  rï  rÐ  s                @@@@@@@@rX   Úadd_to_dtypeÚDCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtypeÊ  sÒ  ÿ€ ð ¤e§h¡h§m¡mð  ¼ÄÇÁÑ8Mô  ð
 ¤u§x¡x§}¡}ð 
 ¼Ä%Ç+Á+Ñ9Nô 
 ð4¬¯©¯©ð 4¼5¿;¹;÷ 4ð
	!¤e§h¡h§m¡mð 	!¼¿¹÷ 	!ð´5·8±8·=±=ð ÄeÇkÁk÷ ð ô # 9§?¡?Ó3ˆOØ)+Ð&Ü(�à—L‘LÐ$:Ó:Ù/°Ó6Ð6˜¼=ÓHô ÕMÀÇÂÓM×MÑMÙ ØŸ*™* Q™-�CØ"×2Ñ2°5Õ9Ø'0×'<Ñ'<Ø&¨c¼%¿+¹+Ð-Fð (=ð (˜ð ×3Ñ3Ø(Ô*Iôô  ×2Ò2°aÑ7Õ2÷ :Ñ9ð —L‘L GÓ+Ù.¨uÓ5Ð5˜¼-ÓGà16·±Ñ.�C˜q¨Qá3°I¸r×BÑBÚ ÜŸG™G×-Ñ-¨dÓ3�EØ"×3Ñ3°EÕ:Ø'0×'<Ñ'<Ø&¨c¸eÐ-Dð (=ð (˜ð ×0Ñ0°¸LÔIä×2Ò2°aÑ7Õ2÷ ;Ò:ð —\‘\ [Ó0ð Ÿ
™
ñØØØ!Ø&Øà ¤MÓ1ð  %Ü!ŸK™KÜ!ŸN™NÜ!ŸM™MÜ!ŸK™Kð	)ó  ð ð  ð  Ø+0´MÓ+AœEŸKšKÀuÜ!ŸK™KØ*Ø!ð&˜Ÿ
ò 2ð& —\‘\ ZÓ/°E·J±J¸r±NÄmÓ4Sà',§z¡zÑ$�S ÜÕMÀÇÂÓM×MÑMÚ Ø"%¬e¯k©kÐ!:�E—JØ—\‘\ ZÓ/°E·J±J¸r±NÄmÓ4Sà#(§:¡:‘L�S˜RÜÕMÀÇÂÓM×MÑMÚ ð /×5Ñ5°eÔ<Ø"%¬%¯+©+Ð!6�E—JØ—\‘\Ð%7Ö7Ø9>¿¹Ñ6�S U¨Ið !¤MÓ1ñ  <¸I×QÑQØ!*×!;Ñ!;¸EÕ!BØ/8×/DÑ/DØ$.°cÀiÐ5Pð 0Eð 0" ð !&× 8Ñ 8¸ÀLÔ Qä '× :Ò :¸aÑ ?Õ :÷ "Cð ¤Ö-ô  ÕUÈÏÊÓU×UÔUà"'§*¡*¨Q¡-˜CØ!*×!:Ñ!:¸5Õ!AØ/8×/DÑ/DØ$.°cÄ%Ç+Á+Ð5Nð 0Eð 0" ð !&× ;Ñ ;Ø$0Ô2Qô!"ô !(× :Ò :¸aÑ ?Õ :÷ "BÒ!Añg )ðz*8¬e¯h©h¯n©n÷ *8ñX ˜yÕ)÷ :×9ú÷$ ;×:ú÷L "CÖ!Bú÷" "B×!Aús2   ÆAWÉ>WÒ>W'Ô9AW9×
W	×
W$	×'
W6	×9
X		)r„   rä  rò  r?  r5  r@  r,  r¬  )rA  rQ  rü  rF  rI  s        rX   Úlegalize_lowp_fp_dtype_loopbodyÚ.CppKernelProxy.legalize_lowp_fp_dtype_loopbodyÉ  sX   € ð^	*¤E§H¡H§N¡Nô ^	*ð@  ×*Ñ*Ð+¬d°9×3FÑ3F×3MÑ3MÓ3OÓ.PÑPˆ
Û#ˆIÙ˜Ÿ™Ö)ò $rW   c                 óþ  ^ • [        U 4S jU 5       5      (       aí  U Hæ  nUR                  R                  /[        UR                  R                  R                  5       5      -   nU H˜  nUR                  R                   H{  nUR                  S;   d  M  UR                  (       d   e[        R                  UR                  ;   d   eUR                  [        R                     nUR                  [        ;   a  M{   e   Mš     Mè     g U Ho  n[        U[        5      (       d   e[        UR                  [         5      (       d   eUR                  nUR#                  5       (       a  M^  T R%                  U5        Mq     g )Nc              3   ót   >#   • U  H-  n[        U[        5      =(       a    TR                  U5      v •  M/     g 7fr  )r¢   r$   r­  )r*  rC  rA  s     €rX   r,  Ú8CppKernelProxy.legalize_lowp_fp_dtype.<locals>.<genexpr>¯  s2   øé € ð 
â�ô �uœmÓ,×Q°×1JÑ1JÈ5Ó1QÔQÚùs   ƒ58)r�  r¶  )r4  r;  r?  r5  r@  r,  r¬  rA  rC  r…  rA   r„  rˆ   rƒ   r¢   r$   r   Úis_memory_copyrþ  )rA  rA  rC  rF  rI  Úfx_noder€  r¡  s   `       rX   Úlegalize_lowp_fp_dtypeÚ%CppKernelProxy.legalize_lowp_fp_dtype®  s8  ø€ Üô 
áó
÷ 
ñ 
ó
 �Ø#Ÿk™k×4Ñ4Ð5¼Ø—K‘K×)Ñ)×0Ñ0Ó2ó9ñ �
ó ",�IØ#,§?¡?×#8Ô#8˜Ø"Ÿ>™>Ð->Õ>Ø#*§<§<Ð/ <Ü#6×#:Ñ#:¸g¿l¹lÓ#JÐJÐ#JØ;B¿<¹<Ü 3× 7Ñ 7ñ<˜Gð $+§=¡=´MÕ#AÐAÐ#Aó $9ó ",ñ	 ð ãˆEÜ˜e¤]×3Ñ3Ð3Ð3Ü˜eŸk™k¬8×4Ñ4Ð4Ð4Ø"Ÿ[™[ˆDØ×&Ñ&×(Ó(Ø×4Ñ4°TÖ:ò rW   c           	      ó�  ^^^^^ ^!• [        T5      [        T5      :X  d   eU R                  m[        TS S9u  mm U R                  TT 5        UU!4S jnUUU U4S jm!U" U R                  5      n[
        R                  =R                  UR                  -  sl        [
        R                  =R                  UR                  -  sl        [        R                  U5      U l        U R                  (       a  U R                  (       d6  U/U l        U R                  SS 5        U R                  R!                  U 5        g ["        R$                  R&                  R)                  SS9   [+        5       nUR-                  TT5      u  pg[        U5      [        U5      :X  d   eSnS n	U(       a…  Sn
US   nUS-   n[        U R                  R.                  5      U:”  aV  U R                  R.                  U   R0                  n
U R                  R.                  U   R0                  nU
=(       a    U(       + n[        U5      S:X  Ga  [2        =R4                  S-  sl        U R                  R7                  US   US   S	9nU" U R8                  US   US   5      nUR:                  UR<                  -
  nUR>                  SUR<                  40Ul         [&        RB                  RD                  (       a  U" U R8                  US   US   U5      nOUnUR>                  /Ul#        UR>                  UR<                  UR:                  40Ul         UU/U l        Un	GO·[        U5      S
:X  GaŸ  US   [        U R                  5      S-
  :X  a  US   US   :X  d   e[2        =R4                  S
-  sl        U R                  R7                  US   US   S	9nSUR<                  4UR<                  UR:                  4S.nUR:                  UR<                  -
  nU R                  R7                  US   US   S	9nSUR<                  4UR<                  UR:                  4S.nUR:                  UR<                  -
  nU" U RH                  US   U5      nUR>                  US   UR>                  US   0Ul         / n[&        RB                  RD                  (       ap  S Hi  u  nnUS:X  a  UOS nUS:X  a  UOS nU" U RH                  US   UUU5      nUR>                  UU   UR>                  UU   0Ul         URK                  U5        Mk     O¾U" U R8                  US   US   5      nUR>                  US   UR>                  US   0Ul         UR>                  /Ul#        URK                  U5        UR>                  US   UR>                  SUR:                  40Ul         UR>                  UR>                  /Ul#        URK                  U5        U/U-   U l        Un	OU/U l        U R                  X‰5        U R                  R!                  U 5        S S S 5        g ! , (       d  f       g = f)Nc                 ó   • [        U S   5      $ ró   rU  rV  s    rX   r½  Ú2CppKernelProxy.codegen_functions.<locals>.<lambda>Ï  r‡  rW   rY  c                 ó¦   >• TR                   " U /UQ76  n[        =R                  S-  sl        T" U5        UsS S S 5        $ ! , (       d  f       g = fró   )Ú
new_kernelr   Úgenerated_kernel_count)r7  r›  rr  rn  Úruns      €€rX   Úcodegen_kernelÚ8CppKernelProxy.codegen_functions.<locals>.codegen_kernelÓ  sA   ø€ Ø×(Ò(¨Ð4¨tÓ4¸ô ×.Ò.°!Ñ3Õ.á�F”Ø÷ 5×4×4ús   •#AÁ
Ac           	      ór  >• U R                  TT5      u  pSn[        TT	5       H~  u  pEUTT4[        [        R                  " TT5      5      S44;   a  U(       a   eU" X5        MB  SnUTS4:X  d   SU ST ST 35       eU R                  5          U" US5        S S S 5        M€     g ! , (       d  f       M’  = f)NFrV   Tzunexpected group: r¶  rœ   )rú  r]  r£   r‰  rŠ  rK  )
rr  r  rs  Ú	in_suffixr�  r‘  ro  r6  r7  rp  s
         €€€€rX   r  Ú-CppKernelProxy.codegen_functions.<locals>.runÝ  sÙ   ø€ Ø#)×#4Ñ#4°U¸OÓ#LÑ ˆDØˆIÜ!$ W¨nÖ!=‘�ØØ˜OÐ,Üœ9Ÿ?š?¨5°/ÓBÓCÀRÐHð!ó ö  )Ð(˜=Ù�tÖ,à $�IØ$ØØð)ó ð Vð ,¨I¨;°d¸5¸'ÀÀOÐCTÐUóVð ð
  ×/Ñ/Õ1Ù˜4 œ÷ 2Ñ1ò ">÷ 2Ö1ús   Â
B'Â'
B6	F©Úinplace_buffersr   r5   )r   r   ©ÚmainÚtailr  )r  )r  r  )r  r  r  )&rÓ   rn  r`   rú  rž  r4   r¬  Úremoved_buffersÚinplaced_to_remover]  r8  r_  r£  r  r¤  Úaggregate_reduction_buffersÚ
set_kernelr„   Ú	_inductorr   ÚpatchrL  r}  rX  rW  r   Úgenerated_cpp_vec_kernel_countÚtilerŸ  rO  Ú
tiled_sizer§   rL  rð  Úenable_loop_tail_vecrM  r   r  )"rA  ro  rp  r  Úscalar_kernelÚtiling_selectÚtiling_factorsr  Ú_inner_loop_reduction_outer_notÚ_outer_loopÚinner_loop_reductionÚouter_loop_levelÚinner_loop_levelÚouter_loop_reductionr  Ú
vec_kernelr$  Útail_kernelÚ
outer_loopri  r  Ú
inner_loopÚinner_rangesr  Útile2d_kernelÚouter_rÚinner_rÚ_inner_tail_sizeÚ_outer_tail_sizerr  r6  rn  r7  r  s"    ``                           @@@@rX   Úcodegen_functionsÚ CppKernelProxy.codegen_functionsÌ  sø  ý€ Ü�7‹|œs >Ó2Ó2Ð2Ð2Ø×(Ñ(ˆÜ!$ ^Ñ9TÑ!UÑˆˆà�‰˜˜Ô/ö	÷	%ð 	%ñ( ' t§¡Ó7ˆÜ	�‰×Ò =×#@Ñ#@Ñ@ÕÜ	�‰×"Ò" m×&FÑ&FÑFÕ"Ü!Ÿ™¨Ó6ˆŒà×"×"¨$¯-¯-Ø)˜?ˆDŒLØ×,Ñ,¨U°DÔ9Ø�N‰N×%Ñ% dÔ+Øô �_‰_×#Ñ#×)Ñ)¸%Ð)Ò@Ü(›NˆMØ-:×-HÑ-HØ˜ó.Ñ*ˆNô �~Ó&¬#¨nÓ*=Ó=Ð=Ð=Ø.3Ð+ØˆKÞØ',Ð$Ø#1°!Ñ#4Ð Ø#3°aÑ#7Ð Ü�t—~‘~×+Ñ+Ó,Ð/?Ó?Ø+/¯>©>×+?Ñ+?Ø(ñ,ç"‘lð )ð ,0¯>©>×+?Ñ+?Ø(ñ,ç"‘lð )ð -×IÐ5IÔ1Ið 4ô �>Ó" aÔ'ä×6Ò6¸!Ñ;Õ6Ø—~‘~×*Ñ*¨>¸!Ñ+<À^ÐTUÑEVÐ*ÐW�Ù+Ø×'Ñ'¨¸Ñ):¸NÈ1Ñ<Mó�
ð !ŸI™I¨¯©Ñ7�	Ø,0¯H©H°q¸$¿/¹/Ð6JÐ+K�
Ô(Ü—:‘:×2×2Ù"0Ø×+Ñ+Ø& qÑ)Ø& qÑ)Ø!ó	#‘Kð #0�KØ48·H±H°:�MÔ0Ø-1¯X©X¸¿¹ÈÏÉÐ7SÐ,T�Ô)Ø *¨KÐ8�”Ø"’Ü�^Ó$¨Ô)à" 1Ñ%¬¨T¯]©]Ó);¸aÑ)?Ó?Ø& qÑ)¨^¸AÑ->Ó>ðð?ô
 ×6Ò6¸!Ñ;Õ6Ø!Ÿ^™^×0Ñ0Ø" 1Ñ%¨n¸QÑ.?ð 1ð �
ð  
× 5Ñ 5Ð6Ø'×2Ñ2°J·O±OÐDñ �ð #-§/¡/°J×4IÑ4IÑ"I�Ø!Ÿ^™^×0Ñ0Ø" 1Ñ%¨n¸QÑ.?ð 1ð �
ð  
× 5Ñ 5Ð6Ø'×2Ñ2°J·O±OÐDñ �ð #-§/¡/°J×4IÑ4IÑ"I�Ù .Ø×*Ñ*Ø" 1Ñ%Ø"ó!�ð —N‘N L°Ñ$8Ø—N‘N L°Ñ$8ð/�Ô+ð !�Ü—:‘:×2×2ó-Ñ(˜ ð 07¸&Ó/@™OÀdð )ð 07¸&Ó/@™OÀdð )ñ "0Ø ×2Ñ2Ø*¨1Ñ-Ø*Ø,Ø,ó"˜ð 'ŸN™N¨L¸Ñ,AØ&ŸN™N¨L¸Ñ,Að0˜Ô,ð $×*Ñ*¨6Ö2ò--ñ0 "0Ø×+Ñ+¨^¸AÑ->ÀÈqÑ@Qó"�Jð #Ÿ™¨°VÑ(<Ø"Ÿ™¨°VÑ(<ð0�JÔ,ð 2<·±Ð0@�JÔ-Ø×&Ñ& zÔ2à"Ÿ™¨°VÑ(<Ø"Ÿ™¨¨J¯O©OÐ(<ð3�MÔ/ð 5?·N±NÀJÇNÁNÐ3S�MÔ0Ø×&Ñ& }Ô5Ø -˜°Ñ<�”Ø(‘à -˜�”Ø×,Ñ,Ø/ôð �N‰N×%Ñ% dÔ+÷} A×@Ö@ús   Å"SX7Ø7
Yc                 ó†   • U H*  nU R                  U5        [        R                  " U5        M,     U R                  X5        g r  )rþ  r:   Úpropagate_loopbodyr5  )rA  rq  rp  r¡  s       rX   Úcodegen_loop_bodiesÚ"CppKernelProxy.codegen_loop_bodies€  s8   € ÛˆDØ×0Ñ0°Ô6Ü×2Ò2°4Ö8ñ  ð 	×Ñ˜{Õ;rW   rA  c                 óð  • U R                  U5        U R                  U5        [        U5      S:¼  d   eS nU Vs/ s H  n[        R                  " X#5      PM     nn[        [        R                  [        5      (       a9  [        R                  R                  (       a  S nU Vs/ s H
  o%" U5      PM     nnU Vs/ s H  o3R                  S   PM     nnU R                  XF5        g s  snf s  snf s  snf )Nr5   c                 óÈ   • U R                  5         U R                  5         [        [        R                  [
        5      (       a  U R                  " U6 $ U R                  U5      $ r  )Údecide_inplace_updateÚmark_runr¢   r4   rr  r1   r;  Úcodegen)r+  Ú
index_varss     rX   r�  Ú(CppKernelProxy.codegen_nodes.<locals>.fnŒ  sG   € Ø×&Ñ&Ô(Ø�M‰MŒOÜœ!Ÿ(™(Ô$5×6Ñ6Ø—z’z :Ð.Ð.à—|‘| JÓ/Ð/rW   c                 óP   • [         R                  R                  U 5      nXl        U$ r  )r4   r*  Úlocalize_functionÚoriginal_fn)r�  Ú
wrapped_fns     rX   Úwrap_fnÚ-CppKernelProxy.codegen_nodes.<locals>.wrap_fn›  s)   € Ü×3Ñ3×EÑEØó�
ð *,Ô&Ø!Ð!rW   )r  r©  rÓ   r`  Úpartialr¢   r4   r*  rK   r+  r6  r5  )rA  rA  r�  r+  ro  rF  rp  s          rX   Úcodegen_nodesÚCppKernelProxy.codegen_nodes†  sÏ   € à×#Ñ# EÔ*Ø×"Ñ" 5Ô)Ü�5‹z˜Q‹Ðˆò	0ñ <AÓAº5°4”9×$Ò$ RÖ.¹5ˆÐAô ”q×-Ñ-Ô/A×BÑBÜ×&Ñ&×4×4ò"ñ .5Ó5ªW r�w˜r–{©WˆGÐ5á49Ó:²E¨DŸ*™* Qœ-±EˆÐ:Ø×Ñ˜wÕ7ùò# Bùò 6ùâ:s   » C)Â&C.Â=C3c                 ó<   • U R                  U R                  X5        g r  )r5  r_  )rA  r³   r  s      rX   r9  ÚCppKernelProxy.codegen_loops§  s   € Ø×Ñ §¡°ÕBrW   c                 óJ   • U R                    H  nUR                  5         M     g r  )r¤  rs  ©rA  rr  s     rX   rs  Ú4CppKernelProxy.update_stores_with_parallel_reductionª  s   € Ø—l”lˆFØ×8Ñ8Ö:ò #rW   r³   c                 óD  • Uc   eSnU R                    Hw  n[        R                  " 5        nUR                  X5      (       a@  SnUR	                  UR                  5       5        UR                  UR                  5       5        S S S 5        My     g ! , (       d  f       M‹  = f)NÚ
C10_LIKELYÚC10_UNLIKELY)r¤  rÀ   rÁ   ra  rÃ   rÄ   r\  r{  )rA  r³   Ú	if_prefixrr  rÌ   s        rX   r{  ÚCppKernelProxy.gen_body®  s{   € ØÑÐÐØ ˆ	Ø—l”lˆFÜ×%Ò%Ô'¨5Ø×,Ñ,¨T×=Ñ=Ø .�IØ×'Ñ'¨¯©«Ô6Ø—K‘K §¡Ó 1Ô2÷	 (Ñ'ò #ß'Ö'ús   ¬ABÂ
B	Úinner_loop_reduction_outer_notr-  Ú	LoopLevelc                 óÖ  ^ • SU 4S jjnT R                   S   nU(       a  U(       d   eU" U5        OZUR                  5         T R                  R                  UR                  5        T R                  R                  UR                  5        T R
                  R                  UR
                  5        T R                  R                  UR                  5        T R                  R                  UR                  5        T R                  R                  UR                  5        T R                  R                  UR                  5        T R                  R                  UR                  5        g)z°
CppKernel/CppVecKernel/CppTile2dKernel have reduction buffers themselves.
Here, we decide how to aggregate them together and place new reduction buffers
under CppKernelProxy.
c           
      ó.  >• [        TR                  5      S:¼  d   eTR                  S   nTR                  S   n[        UTR                  5      (       d   e[	        U5      TR
                  L a^  UR                  UR                  5        UR                  5         TR                  R                  UR                  UR                  -   5        O5UR                  5         TR                  R                  UR                  5        [        5       n[        R                  " 5        nUR                  USU R                  5      (       a:  UR                  UR!                  5       5        UR                  UR"                  5        S S S 5        [        R                  " 5        nUR                  USU R                  5      (       Ga$  UR                  UR!                  5       5        [	        U5      TR
                  L aÒ  UR$                  nU HW  nU SU R                   S['        U R(                  5       S3n[+        UR,                  Xg5        [+        UR"                  Xg5        MY     [/        UR,                  5        UR                  [1        UR"                  U R                  U R                   S	3U R(                  U R2                  5      5        OUR                  UR"                  5        S S S 5        UTl        g ! , (       d  f       GN|= f! , (       d  f       N(= f)
Nr   r   rc  rQ  rR  r×   z_tail - rª  rw  )rÓ   r¤  r¢   rŸ  r(  rž  rÅ  rÓ  rP  r\  r7   rÀ   rÁ   ra  r§   rÃ   rÄ   rR  rc  rE   r   rç   rq  rñ   rÐ   rO  )	r-  Úmain_loop_kernelÚtail_loop_kernelÚ
suffix_bufrÌ   rs  rÇ   rä   rA  s	           €rX   Ú!aggregate_reduction_prefix_suffixÚUCppKernelProxy.aggregate_reduction_buffers.<locals>.aggregate_reduction_prefix_suffixÁ  s”  ø€ Ü�t—|‘|Ó$¨Ó)Ð)Ð)Ø#Ÿ|™|¨A™ÐØ#Ÿ|™|¨BÑ/ÐÜÐ.°×0CÑ0C×DÑDÐDÐDô Ð$Ó%¨¯©Ò8ð !×:Ñ:Ø$×2Ñ2ôð !×:Ñ:Ô<Ø×%Ñ%×,Ñ,Ø$×5Ñ5Ø&×7Ñ7ñ8õð
 !×:Ñ:Ô<Ø×%Ñ%×,Ñ,Ð-=×-NÑ-NÔOô &›ˆJÜ×%Ò%Ô'¨5Ø#×6Ñ6Ø ¨j¯n©n÷ñ ð ×'Ñ'¨
×(9Ñ(9Ó(;Ô<Ø×%Ñ%Ð&6×&GÑ&GÔH÷ (ô ×%Ò%Ô'¨5Ø#×6Ñ6Ø °
·±÷ò ð ×'Ñ'¨
×(9Ñ(9Ó(;Ô<ÜÐ,Ó-°·±Ò@Ø)9×)MÑ)M˜Û$2˜DØ*.¨¨u°Z·^±^Ð4DÀHÌ[ÐYc×YnÑYnÓMoÐLpÐpqÐ'r˜HÜ,Ð-=×-DÑ-DÀdÔUÜ,Ø 0× AÑ AÀ4öñ %3ô 5Ð5E×5LÑ5LÔMØ"×)Ñ)Ü6Ø 0× AÑ AØ *§¡Ø#-§>¡>Ð"2°%Ð 8Ø *× 5Ñ 5Ø *§¡óõð #×)Ñ)Ð*:×*KÑ*KÔL÷5 (ð6 %/ˆDÕ!÷C (Ö'ú÷ (Õ'ús   Ä"AK4ÆELË4
LÌ
Lr   N)r-  rV  )r¤  rÅ  rP  r\  rR  rS  rT  rU  rV  rX  rY  )rA  rU  r-  r\  Úmain_kernels   `    rX   r  Ú*CppKernelProxy.aggregate_reduction_buffers¸  s  ø€ ÷9	/ðv —l‘l 1‘oˆÞ)ÞÐ�:Ù-¨jÕ9à×1Ñ1Ô3Ø×!Ñ!×(Ñ(¨×)EÑ)EÔFØ×!Ñ!×(Ñ(¨×)EÑ)EÔFØ×&Ñ&×-Ñ-¨k×.SÑ.SÔTØ×&Ñ&×-Ñ-¨k×.SÑ.SÔTØ×!Ñ!×(Ñ(¨×)IÑ)IÔJØ×#Ñ#×*Ñ*¨;×+MÑ+MÔNØ×*Ñ*×1Ñ1Ø×5Ñ5ô	
ð 	×*Ñ*×1Ñ1Ø×5Ñ5õ	
rW   )rN  rn  r¤  r_  r£  rR  r  ) r  r  r  r  rF  rž  r(  r   r  rŸ  rB  r   r@  r©  r$   r­  r   rþ  r  r5  r9  r5  rI  r9  rs  r   r7   r{  rl   r  r!  ry  rz  s   @rX   r  r  ¦  sº   ø‡ ð #,€J��Y‘Ó+Ø)5€N�D˜Ñ&Ó5Ø/>Ð�t˜OÑ,Ó>õ+ò@ð
°=ô 
ðc*¸ô c*òJ;ò<r,òh<ð8 4¨Ñ#6ô 8òBCò;ñ3˜X lÑ3õ 3ðU
Ø.2ðU
Ø@HÈÑ@U÷U
ò U
rW   r  c                   ó.   ^ • \ rS rSrU 4S jrS rSrU =r$ )ro  i  c                 óp   >• [         TU ]  UR                  UR                  R                  5        / U l        g r  )r?  r@  r›  r¢  ra  rq  r¥  s     €rX   r@  ÚOuterLoopFusedKernel.__init__  s)   ø€ Ü‰Ñ˜×*Ñ*¨L¯O©O×,GÑ,GÔHØ%'ˆ�
rW   c           
      óž  • / nU R                    Vs/ s H  oDR                  5       PM     nnU Hg  nUR                  nUc   eUR                  UR	                  [        [        U5      UR                  -
  UR                  S9U5      R                  5        Mi     [        [        UR                  [        U5      5      UR                  S9$ s  snf )NrB  )rq  r
  rN  r  r&  r  rÓ   r  r  r_   r`   )rA  r'  r  Úkernels_parallel_depthr_  Únested_kernelsrr  rN  s           rX   r&  Ú*OuterLoopFusedKernel.decide_parallel_depth  sÙ   € Ø!#Ðà48·J²Jó+
Ú4> y× Ñ Ö"±Jð 	ð +
ó %ˆFð !×,Ñ,ˆKØÑ*Ð*Ð*Ø"×)Ñ)Ø×,Ñ,Ü!ä Ó,Ð/A×/MÑ/MÑMà$6×$BÑ$Bñ	ð ó÷ !‘.ö
ñ %ô  ÜØ"×1Ñ1´3Ð7MÓ3Nóð +×6Ñ6ñ	
ð 	
ùò)+
s   ‘C
)rq  )r  r  r  r  r@  r&  r!  ry  rz  s   @rX   ro  ro    s   ø† õ(÷
ð 
rW   ro  c                   ó    • \ rS rSrSrSrSrSrg)ÚReasonFusedNodesi3  Úsame_vars_reduceÚcompatible_reductionÚcompatible_ranges_no_reductionrV   N)r  r  r  r  ÚSAME_VARS_REDUCEÚCOMPATIBLE_REDUCTIONÚCOMPATIBLE_RANGES_NO_REDUCTIONr!  rV   rW   rX   rh  rh  3  s   † Ø)ÐØ1ÐØ%EÓ"rW   rh  c                   óÄ  ^ • \ rS rSr% \r\\   \S'   Sr\	" \
R                  \
R                  /5      r\S\R                   S\	\
   4S j5       rU 4S jrS\4S	 jrS
 rS rS rS\\   4S jrS rS rS rS\S\S\4S jrS r S r!S r"S r#S\$\%   4S jr&S\'4S jr(S\)\'\*\%4   4S jr+S\S\4S jr,S\S\-\   S \-\   4S! jr.S" r/S# r0S$ r1S)S% jr2S& r3S)S' jr4S(r5U =r6$ )*ÚCppSchedulingi9  Úkernel_proxy_clsiô  Údevicer¸   c                 ó   • U R                   $ r  )Úbackend_features)r7  rr  s     rX   Úget_backend_featuresÚ"CppScheduling.get_backend_featuresI  s   € à×#Ñ#Ð#rW   c                 ó`   >• [         TU ]  U5        U(       a  U R                  5         SU l        g rd  )r?  r@  Úreset_kernel_groupÚ_ready_to_flush)rA  r3  rD  s     €rX   r@  ÚCppScheduling.__init__M  s'   ø€ Ü‰Ñ˜Ô#ÞØ×#Ñ#Ô%Ø$ˆÕrW   Ústatusc                 ó   • Xl         g r  ©ry  )rA  r{  s     rX   Ú_set_flush_statusÚCppScheduling._set_flush_statusS  s   € Ø%ÕrW   c                 ó&   • [        S U 5       5      $ )Nc              3   ó�   #   • U  H<  n[        [        [        R                  R                  R
                  U5      5      v •  M>     g 7fr  )r£   rÀ  r4   r¬  rÉ  rø   r‚  s     rX   r,  Ú)CppScheduling.group_fn.<locals>.<genexpr>W  s/   é € ÐMÂuÀ!”Uœ3œqŸw™w×/Ñ/×8Ñ8¸!Ó<×=Ð=Âuùs   ‚AA)r£   )rA  rW  s     rX   Úgroup_fnÚCppScheduling.group_fnV  s   € ÜÑMÁuÓMÓMÐMrW   c                 ó"   • [        5       U l        g r  )ÚKernelGrouprn  rG  s    rX   rx  Ú CppScheduling.reset_kernel_groupY  s   € Ü'›MˆÕrW   c                 óè  ^• UR                  5       (       d  UR                  5       (       a  [        R                  " X5      $ UR                  5       (       a-  UR                  5       (       a   e[        R                  " X5      $ U R                  X5      [        R                  :X  Gaû  [        U[        [        45      (       d   e[        U[        [        45      (       d   eUR                  u  nu  pEUR                  u  nu  pgUS:X  a  US:X  d   XW45       eU4S jm[        U5      [        U5      :  a  UOUn[        U[        5      (       d   e[        U5      [        U5      :  a  UOUn	T" U	5      n
UR                  U
S9  UR                  u  nu  pCUR                  u  nu  pcXF:X  a  [        R                  " X5      $ T" U5      n[        U	[        5      (       a  U	R                  US9  Op[        U	[        5      (       d   eU	R                   H)  n[        U[        5      (       d   eUR                  US9  M+     [	        U	R                  U	R                  5      n	UR                  u  nu  pCUR                  u  nu  pcXF:X  d   XF45       e[        R                  " X5      $ U R                  X5      (       a%  [         R                  XU R#                  X5      5      $ [        R                  " X5      $ )NrV   c                 óf  >• [        U [        5      (       a–  [        U R                  5      S:”  d   U R                  5       eS n[        [
           " 5       nU R                   H;  nT	" U5      u  pEUc  UnX:X  d   XU R                  45       eUR                  U5        M=     U[        U5      4$ [        U [        5      (       d   eU R                  n[        U[        R                  5      (       d   eUR                  5       u  pxnUR                  [        UR                  R                  5       5      4$ rm  )r¢   r"   rÓ   Úsnodesr   r   Úupdater5  r$   r+  r   ÚComputedBufferÚget_default_sizes_bodyr™  Úindexing_exprsr,  )
r+  r™  rŽ  ÚsnodeÚvÚexprsÚcomp_bufferrÍ   r¡  Úget_indexing_ranges_exprss
            €rX   r“  Ú5CppScheduling.fuse.<locals>.get_indexing_ranges_exprsn  s	  ø€ Ü! $Ô(:×;Ñ;Ü" 4§;¡;Ó/°!Ó3Ð@°T·[±[Ó@Ð3Ø%)˜
Ü)3´CªÓ):˜Ø%)§[¤[˜EÙ'@ÀÓ'G™H˜AØ)Ñ1Ø-. 
Ø#-£?ÐP°ZÀDÇKÁKÐ4PÓP ?Ø*×1Ñ1°%Ö8ñ &1ð  *¬4°Ó+?Ð?Ð?ä)¨$´×>Ñ>Ð>Ð>Ø&*§i¡i˜Ü)¨+´r×7HÑ7H×IÑIÐIÐIØ%0×%GÑ%GÓ%I™
˜ Ø#Ÿ™´°T×5HÑ5H×5OÑ5OÓ5QÓ0RÐRÐRrW   )Úextra_indexing_constraints)Ú
is_foreachr!   r9  Úis_templater"   Ú_why_fuse_nodesrh  rn  r¢   r$   r6  rÓ   Úrecompute_size_and_bodyrŠ  r3  Úcan_fuse_vertical_outer_loopr#  Ú_get_outer_loop_fusion_depth)rA  r$  r%  rÍ   Úvars1Úreduce1Úvars2Úreduce2Únode_to_recompÚref_nodeÚref_indexing_constraintsÚ#node_to_recomp_indexing_constraintsr�  r“  s                @rX   r9  ÚCppScheduling.fuse\  s¶  ø€ Ø×Ñ×Ñ ×!1Ñ!1×!3Ñ!3Ü-×2Ò2°5Ó@Ð@Ø×Ñ× Ñ Ø×(Ñ(×*Ñ*Ð*Ð*Ü%×*Ò*¨5Ó8Ð8ð ×$Ñ$ UÓ2Ü#×BÑBôCô " %¬-Ô9KÐ)L×MÑMÐMÐMÜ! %¬-Ô9KÐ)L×MÑMÐMÐMà&+§k¡kÑ#�Ñ#�EØ&+§k¡kÑ#�Ñ#�EØ "“}¨°B«ÐJ¸Ð8JÓJÐ6õSô& +.¨e«*´s¸5³zÓ*A¡Àu�Ü! .´-×@Ñ@Ð@Ð@ä$'¨£J´°U³Ó$;™5À�á+DÀXÓ+NÐ(à×6Ñ6Ø/Gð 7ñ ð !&§¡‘�‘:�EØ %§¡‘�‘:�Eà“>Ü-×2Ò2°5Ó@Ð@ñ 7PØ"ó7Ð3ô ˜h¬×6Ñ6Ø×4Ñ4Ø3Vð 5ò ô & hÔ0B×CÑCÐCÐCØ!)§¤˜Ü)¨%´×?Ñ?Ð?Ð?Ø×5Ñ5Ø7Zð 6ó ñ "1ô
  2°(×2DÑ2DÀhÇoÁoÓV�Hà %§¡‘�‘:�EØ %§¡‘�‘:�EØ“~Ð5¨ ~Ó5�~Ü)×.Ò.¨uÓ<Ð<Ø×2Ñ2°5×@Ñ@Ü2×7Ñ7Ø $×"CÑ"CÀEÓ"Qóð ô *×.Ò.¨uÓ<Ð<rW   c                 ó  • UR                   u  nu  pEUR                   u  nu  pgXF:X  a  XW:X  a  [        R                  $ US:X  a  XFU-   :X  a  [        R                  $ U R	                  X5      (       a  [        R
                  $ g )NrV   )r6  rh  rl  rm  Ú&_can_fuse_nodes_with_compatible_rangesrn  )rA  r$  r%  rÍ   rœ  r�  rž  rŸ  s           rX   r˜  ÚCppScheduling._why_fuse_nodes®  sv   € Ø#Ÿk™kÑˆÑˆEØ#Ÿk™kÑˆÑˆEà‹>˜gÓ0Ü#×4Ñ4Ð4Ø�b‹=˜U¨g¡oÓ5Ü#×8Ñ8Ð8Ø×6Ñ6°u×DÑDÜ#×BÑBÐBàrW   c                 óÔ  • UR                   u  nu  pEUR                   u  nu  pgUS:H  =(       a    US:H  n[        R                  " U5      [        R                  " U5      :H  n	[        U5      S:H  =(       d    [        U5      S:H  n
U(       a  U	(       a  U
(       d  g[        U5      [        U5      :  a  UOUn[        U5      [        U5      :  a  UOUn[	        U[
        5      (       a  g[	        U[        5      (       d   e[	        UR                  [        R                  5      (       a  g[	        UR                  [        R                  5      (       d   eUR                  R                  R                  5       nS n[	        U[
        5      (       aî  [        [        [        S4      " 5       nUR                    H•  n[	        UR                  [        R                  5      (       a    Ok[	        UR                  [        R                  5      (       d   eUR#                  [        UR                  R                  R                  5       5      5        M—     [        U5      S:w  a  g[%        ['        [)        U5      5      5      nOf[	        U[        5      (       d   e[	        UR                  [        R                  5      (       d   eUR                  R                  R                  5       nXí:w  a  gg)NrV   r5   F.T)r6  ry  re   rÓ   r¢   r"   r$   r+  r   ÚTemplateBufferrŒ  ÚdataÚget_sizer   r£   r   rŠ  r¦  r5  ÚnextÚiter)rA  r$  r%  rÍ   rœ  r�  rž  rŸ  Úc1Úc2Úc3r   r¡  Úranges2Úranges1Ú
ranges_setr�  s                    rX   r¦  Ú4CppScheduling._can_fuse_nodes_with_compatible_ranges»  s+  € ð $Ÿk™kÑˆÑˆEØ#Ÿk™kÑˆÑˆEà˜‰]×,˜w¨"™}ˆÜ�YŠY�uÓ¤§¢¨5Ó!1Ñ1ˆÜ�‹Z˜1‰_×/¤ E£
¨a¡ˆÞ–ržbØä"% e£*¬s°5«zÓ"9™¸uˆÜ ›J¬¨U«Ó3‘5¸ˆô �nÔ&8×9Ñ9Øô ˜.¬-×8Ñ8Ð8Ð8Ü�n×)Ñ)¬2×+<Ñ+<×=Ñ=ØÜ˜.×-Ñ-¬r×/@Ñ/@×AÑAÐAÐAð !×%Ñ%×*Ñ*×3Ñ3Ó5ˆØˆÜ�hÔ 2×3Ñ3Ü#¤E¬#¨s¨(¡OÒ4Ó6ˆJØ!Ÿœ�Ü˜eŸj™j¬"×*;Ñ*;×<Ñ<ÙÜ! %§*¡*¬b×.?Ñ.?×@Ñ@Ð@Ð@Ø—‘œu U§Z¡Z§_¡_×%=Ñ%=Ó%?Ó@ÖAñ	 )ô �:‹ !Ó#Øäœ4¤ ZÓ 0Ó1Ó2‰Gä˜h¬×6Ñ6Ð6Ð6Ü˜hŸm™m¬R×->Ñ->×?Ñ?Ð?Ð?Ø—m‘m×(Ñ(×1Ñ1Ó3ˆGàÓØàrW   c                 óÎ   • [        U[        [        45      (       d   e[        U[        [        45      (       d   e[        S X4 5       5      (       a  gU R	                  X5      S L$ )Nc              3   óB   #   • U  H  n[        U[        5      v •  M     g 7fr  )r¢   r#  r)  s     rX   r,  Ú:CppScheduling._can_fuse_horizontal_impl.<locals>.<genexpr>ö  s   é € ð 
ÚFT¸dŒJ�tÔ8×9Ð9Ânùs   ‚F)r¢   r"   r$   rg   r˜  ©rA  r$  r%  s      rX   Ú_can_fuse_horizontal_implÚ'CppScheduling._can_fuse_horizontal_impló  sk   € Ü˜%Ô"4´mÐ!D×EÑEÐEÐEÜ˜%Ô"4´mÐ!D×EÑEÐEÐEÜñ 
ØGLÁnó
÷ 
ñ 
ð Ø×#Ñ# EÓ1¸Ð=Ð=rW   c                 ó  • UR                  5       (       d  UR                  5       (       a  g[        UR                  5       5      [        UR                  5       5      -   [        R                  R
                  :”  a  gU R                  X5      $ rd  )r—  rÓ   r>  r   rð  Úmax_horizontal_fusion_sizer¹  r¸  s      rX   Úcan_fuse_horizontalÚ!CppScheduling.can_fuse_horizontalü  sj   € Ø×Ñ×Ñ %×"3Ñ"3×"5Ñ"5Øä�—‘Ó!Ó"¤S¨¯©Ó):Ó%;Ñ;Ü�j‰j×3Ñ3ó4ð à×-Ñ-¨eÓ;Ð;rW   r$  r%  c                 ó˜  • UR                  5       =n(       a³  [        UR                  [        R                  5      =(       aˆ    [        UR
                  [        R                  5      =(       a]    [        UR
                  R                  5      S:H  =(       a4    UR
                  R                  S   R                  5       UR                  :H  $ g)Nr5   r   F)Úget_template_noder¢   Úlayoutr   ÚMultiOutputLayoutr+  ÚMultiOutputrÓ   Úinputsr.  rÇ   )rA  r$  r%  Útemplate_bufs       rX   Úcan_fuse_multi_outputs_templateÚ-CppScheduling.can_fuse_multi_outputs_template  s›   € ð !×2Ñ2Ó4Ð4ˆ<Õ4ä˜<×.Ñ.´×0DÑ0DÓE÷ IÜ˜uŸz™z¬2¯>©>Ó:÷Iä˜Ÿ
™
×)Ñ)Ó*¨aÑ/÷Ið —J‘J×%Ñ% aÑ(×1Ñ1Ó3°|×7HÑ7HÑHð	ð rW   c                 ó:  • Sn[        S X4 5       5      (       d  U$ [        U[        5      (       a  UR                  5       S   OUn[        U[        [
        45      (       d   e[        U[        5      (       a  UR                  5       S   OUn[        U[        [
        45      (       d   eUR                  u  nu  pxUR                  u  nu  pšUS:X  a  U	S:X  a  US:w  a  U
S:w  a  U$ [        S X4 5       5      (       a(  UR                  UR                  :X  a  UR                  $ U$ [        [        U5      [        U	5      5      nUS:¼  aP  US U U	S U :X  aD  [        S X4 5       5      (       a*  [        U5      [        L a  UOUnUR                  U:X  a  U$ U$ U$ U$ )Nr   c              3   ó\   #   • U  H"  n[        U5      [        [        [        4;   v •  M$     g 7fr  )r(  r#  r"   r$   r)  s     rX   r,  Ú=CppScheduling._get_outer_loop_fusion_depth.<locals>.<genexpr>  s.   é € ð 
ò '�ô �‹JÜ+Ô-?ÄÐOöPâ&ùr.  rc  rV   c              3   óD   #   • U  H  n[        U5      [        L v •  M     g 7fr  r0  r)  s     rX   r,  rÊ  .  r1  r2  r5   c              3   óD   #   • U  H  n[        U5      [        L v •  M     g 7fr  r0  r)  s     rX   r,  rÊ  9  s   é € ð ÚFT¸d”�T“
Ô9Õ9Ânùr2  )r4  r¢   r#  r6  r"   r$   r6  r8  r_   rÓ   rg   r(  )rA  r$  r%  ÚDISABLE_OUTER_LOOP_FUSIONÚ_node1Ú_node2rÍ   rœ  r�  rž  rŸ  r8  Ú_compare_nodes                rX   r›  Ú*CppScheduling._get_outer_loop_fusion_depth  sÖ  € Ø$%Ð!Üñ 
ð ™ó
÷ 
ñ 
ð
 -Ð,ô ˜%Ô!<×=Ñ=ð ×!Ñ!Ó# BÒ'àð 	ô
 ˜&Ô#5´}Ð"E×FÑFÐFÐFô ˜%Ô!<×=Ñ=ð ×!Ñ!Ó# AÒ&àð 	ô
 ˜&Ô#5´}Ð"E×FÑFÐFÐFà$Ÿl™lÑˆÑˆEØ$Ÿl™lÑˆÑˆEØ�B‹;˜5 B›;¨7°b«=¸WÈ»]à,Ð,ÜÑTÀeÁ^ÓT×TÑTð ×0Ñ0°E×4QÑ4QÓQð ×-Ñ-ðð /ðô
 #&¤c¨%£j´#°e³*Ó"=Ðà# qÓ(ØÐ.Ð.Ð/°5Ð9QÐ:QÐ3RÓRäñ ØGLÁnó÷ ñ ô " %›[Ô,GÒG‘EÈUð ð !×8Ñ8Ð<SÓSà2Ð2à4Ð4ð /Ð.Ø(Ð(rW   c                 óT  • UR                  5       (       + =(       a�    UR                  5       (       + =(       aq    UR                  5       UR                  -  =(       aM    U R                  X5      =(       a    UR	                  5       (       + (       + =(       a    U R                  X5      S:¬  $ ró   )r—  Úget_operation_namesÚ	ancestorsr¹  rW  r›  r¸  s      rX   rš  Ú*CppScheduling.can_fuse_vertical_outer_loopI  s�   € à×!Ñ!Ó#Ô#÷ EØ×%Ñ%Ó'Ô'÷Eà×)Ñ)Ó+¨e¯o©oÑ=÷Eð ×.Ñ.¨uÓ<÷ -Ø×*Ñ*Ó,Ô,ô÷Eð ×1Ñ1°%Ó?À1ÑDð		
rW   c                 ó2   • U R                  X5      (       a  ggrc  )rš  r¸  s      rX   Úget_fusion_pair_priorityÚ&CppScheduling.get_fusion_pair_priorityU  s   € Ø×,Ñ,¨U×:Ñ:ààrW   c                 ó:  • UR                  5       (       a  gUR                  5       (       a,  [        X/5      u  p4UR                  5       (       + =(       a    U$ U R                  X5      =(       a    UR                  5       (       + =(       d    U R	                  X5      $ rd  )r—  rN   rW  r¹  rš  )rA  r$  r%  Útemplate_fusion_supportedrÍ   s        rX   Úcan_fuse_verticalÚCppScheduling.can_fuse_vertical\  s‡   € Ø×Ñ×ÑàØ×Ñ×ÑÜ+SØ�wó,Ñ(Ð%ð ×)Ñ)Ó+Ô+×IÐ0IÐIà×*Ñ*¨5Ó8×UÀ×ASÑASÓAUÔ=U÷=à×.Ñ.¨uÓ<ð	=rW   rA  c                 óX  ^^^^^• [        S U 5       5      (       a  U$ SmSmSnSnSnSnSn/ nU GHÚ  n[        UR                  [        R                  5      (       d   eUR                  R                  5       n	UR                  X‰45        U	u  u  p«pËUR                  R                  5        GHY  u  mn[        U[        R                  5      (       d  M(  UR                  [        5       GH  m[        U4S jUR                   5       5      (       a  TU:w  a  TnUS-  nUS:”  a  Us  s  s  $ [        TR                  S   [        R                  R                   R"                  5      (       d  M…  TR                  S   UR                  ;   d  M¤  Tc  M©  [%        UU4S jUR                  R                  5        5       5      (       d  MÞ  TR                  S   S:”  d  Mó  TR                  S   mTR                  S   mS	nUnU
nGM     GM\     GMÝ     U(       d  U$ Uc   e['        U5      nU H2  u  nu  u  p«pËTUR                  ;  a  Us  $ ['        U
5      U:w  d  M0  Us  $    SmUUU4S
 jnU H  nX…:X  d  M
  UR)                  US9  M     U H  nX…:w  d  M
  UR)                  TUS9  M     U$ )aÙ  
Apply loop split optimization.
When one of the indexing_exprs contains a division, we eliminate the division by splitting the loop
to avoid non-contiguous loads, subject to the following conditions:
    1. No reduction and no mudular index for all nodes.
    2. The indexing_exprs of all nodes contain only one (or more, but all the same) division,
       where the divisor is an integer and not too small (the divisor > 8), the dividend is
       one of the iter_vars, and this var, i.e. the dimension that needs to be split, is
       contiguous in all other indexing_exprs.

For example, if the node's var_ranges: {z0: 2, z1: 9216, z2: 960} and indexing_exprs:
{'index0': 8847360*z0 + 960*z1 + z2, 'index1': 32*z0 + (z2//30), 'index2': z2},
we will split z2 -> 30*z2 + z3, then the node's var_ranges will be changed to
{z0: 2, z1: 9216, z2: 32, z3: 30} and indexing_exprs will be changed to
{'index0': 8847360*z0 + 960*z1 + 30*z2 + z3, 'index1': 32*z0 + z2, 'index2': 30*z2 + z3}.
c              3   óÔ   #   • U  H^  n[        UR                  S    S    5      S:g  =(       d4    [        S UR                  R                  R                  5        5       5      v •  M`     g7f)r5   r   c              3   óJ   #   • U  H  oR                  [        5      v •  M     g 7fr  )rô   r   )r*  r¹  s     rX   r,  Ú9CppScheduling.try_loop_split.<locals>.<genexpr>.<genexpr>~  s   é € ð Ú6X¨d—‘œ×)Ð)Ò6Xùs   ‚!#N)rÓ   r6  rg   r;  rŽ  r,  r)  s     rX   r,  Ú/CppScheduling.try_loop_split.<locals>.<genexpr>|  sd   é € ð 
ò
 �ô	 �—
‘
˜1‘˜aÑ Ó! QÑ&÷ Üñ Ø6:·j±j×6OÑ6O×6VÑ6VÔ6Xóó ôò ùs   ‚A&A(Nr   Fc              3   óF   >#   • U  H  nTR                  U5      v •  M     g 7fr  )rô   )r*  r§   Údiv_exprs     €rX   r,  rá  š  s   øé € ÐQÒ9P°#˜HŸL™L¨×-Ð-Ò9PùrS  r5   c              3   óp   >#   • U  H+  u  pUT:w  d  M  [        UTR                  S    5      S;   v •  M-     g7f)r   rf  N)r  r›  )r*  Úname_Úexpr_rã  rÇ   s      €€rX   r,  rá  ¥  s:   øé € ð  â0T¡ Ø$¨™}ó SÔ/°°x·}±}ÀQÑ7GÓHÈFÖRÚ0Tùs   ƒ6“#6rx  Tc                 óÔ  >• U u  p4Uu  pVUR                  T5      nUR                  5       nX7   T-  X‡'   UR                  US-   T5        [        R                  " X„SS9u  u  pšnU	R                  5       nUR                  US-   5      nTXÇ   -  U-   XÇ'   [        R                  " XU/X¹U5      nT(       d/  UR                  [        UR                  R                  5       5      4mX„4UX–44$ )Nr5   r=  )r  )r’   ÚcopyÚinsertr
   Úindex_vars_no_squeezeÚpopr   r   r™  r5  rŽ  r,  )rW  r¡  r  Ú
index_sizeÚreduce_sizer@  Úreduce_varsÚ	split_idxÚnew_index_sizeÚnew_index_varsrÍ   r™  Ú	iter_varsÚdivisor_varr•  Úsplit_numberÚ	split_vars                 €€€rX   Ú
loop_splitÚ0CppScheduling.try_loop_split.<locals>.loop_splitÅ  s  ø€ Ø&+Ñ#ˆJØ&*Ñ#ˆJØ"×(Ñ(¨Ó3ˆIØ'Ÿ_™_Ó.ˆNØ(2Ñ(=ÀÑ(MˆNÑ%Ø×!Ñ! )¨a¡-°Ô>Ü.:×.PÒ.PØ°Cñ/Ñ+Ñˆ^ ð '×+Ñ+Ó-ˆIØ#Ÿ-™-¨	°A©Ó6ˆKØ#/°)Ñ2FÑ#FÈÑ#TˆIÑ Ü—;’;Ø +Ð.°
ÈKóˆDö .à—O‘OÜ˜×,Ñ,×3Ñ3Ó5Ó6ð.Ð*ð
  Ð-ØØÐ-ðð rW   )Úrecompute_sizes_body_func)r•  rø  )rg   r¢   r+  r   rŒ  r�  r  rŽ  r  rõ   rý  Úfindr   rò  r›  ÚcoreÚnumbersre  r4  rÓ   r™  )rA  rA  Únum_divÚ	div_expr_Ú	match_divÚmatched_nodeÚmatched_index_sizeÚnode_bodiesr+  Ú
sizes_bodyrì  rÍ   Úoriginal_bodyr¹  Úmatched_num_dimsrö  rã  r•  rÇ   rô  rõ  s                   @@@@@rX   Útry_loop_splitÚCppScheduling.try_loop_spliti  s•  ü€ ô& ñ 
ñ
 ó
÷ 
ñ 
ð ˆLàˆ	ØˆØˆØˆ	Øˆ	ØˆØ!Ðð .0ˆäˆDÜ˜dŸi™i¬×):Ñ):×;Ñ;Ð;Ð;ØŸ™×9Ñ9Ó;ˆJØ×Ñ Ð1Ô2Ø0:Ñ-‰OˆZ˜]Ø+×:Ñ:×@Ñ@×B‘
��dÜ! $¬¯
©
×3Ñ3ÙØ $§	¡	¬(× 3�HäÔQ¸×9PÒ9PÓQ×QÑQØ$¨	Ó1à$,˜	Ø 1™˜Ø “{Ø$žä" 8§=¡=°Ñ#3´U·Z±Z×5GÑ5G×5OÑ5O×PÓPØ$ŸM™M¨!Ñ,°×0GÑ0GÕGØ Ó,Üõ  à0=×0LÑ0L×0RÑ0RÔ0Tó ÷ ó ð
 %ŸM™M¨!Ñ,¨qÕ0à$,§M¡M°!Ñ$4˜	Ø'/§}¡}°QÑ'7˜Ø$(˜	Ø'+˜Ø-7Ó*ô1 !4ô Cñ öF ØˆLð "Ñ-Ð-Ð-ÜÐ1Ó2Ðã9DÑ5ˆDÑ5‘?�J MØ × 7Ñ 7Ó7Ø’Ü�:‹Ð"2Õ2Ø’ñ	 :Eð &*Ð"÷	ó: ˆDØÕ#Ø×,Ñ,ÀzÐ,ÓRñ ó ˆDØÕ#Ø×,Ñ,Ø/IØ.8ð -ó ñ ð ˆrW   r+  c                 ó~  ^ ^^^	• T R                   m[        R                  n/ m/ m	[        U[        5      (       d   eS[        4UUU	U 4S jjnU" U5      (       dÓ  U[        l        TR                  5         T	R                  5         [        R                  R                  R                  SS9   UR                  5        Hc  n[        U[        [        45      (       d   eUR                  5       nT R                  T5      nUR                  U5        TR!                  Xe5        Me     SSS5        gg! , (       d  f       g= f)zð
Generate the code for the outer loop fused scheduler node.
1. Codegen with fused outer loop: depends on the analysis of
    the outer loop fused scheduler node, with or without the local buffer.
2. If failed, fallback to standard codegen.
r+  c           	      ó^	  >^ ^^^^^• [        T [        5      (       d   eTR                  5         TR                  5         S[        4S jm/ n0 m[	        UU 4S jT R                  5        5       5      (       Ga6  [        5       mT R                  5        GH  m[        T[        5      (       d   eTR                  TR                  5       5        TR                  5       (       d  [        TR                  5       5      S:w  a  Mn  TR                  5       S   m[	        U 4S jTR                   5       5      (       d  M§  TR                  n[        U[         R"                  5      (       d   eUR%                  5       nT R&                  [        T" T5      5      -
  nUU4S jnUR)                  5       (       a  U" 5       (       d  GM-  / nUR*                  S   nT" T5      US	 n[-        U5       H  n	UR/                  SU5        Xy-  nM     [         R0                  " UR2                  UR4                  UU5      n
UU4S
 jnSnU" X¡5      nU(       dC  [         R6                  " U S[        U5       3U
S9nUR9                  U5        / TUR:                  '   TUR:                     R9                  U5        GM     [=        TR>                  5       n[        U5      S:”  a7  U H1  nUR:                  c   eURA                  UTUR:                     5        M3     T R                  5        H€  n[        U[B        [        45      (       d   eTRE                  T5      nURG                  UR                  5       5        TR9                  U5        TR9                  UR                  5       5        M‚     T RI                  TT R&                  5      (       dF  URJ                   H,  n[L        RN                  RJ                  RQ                  U5        M.      S	S	S	5        g[R        RT                  R9                  [R        RV                  " [        T5      [        URX                  5      S95        T R[                  T5      nTR]                  U/ [^        R`                  Rc                  T5      Q5        S	S	S	5        g! , (       d  f       g= f)z6
Codegen code with fused outer loop and local Buffer.
r+  c                 óÂ   • [        U [        [        45      (       d   eU R                  5       n[	        US S9R
                  u  nu  p4[        U5      [        U5      -   nU$ )Nc                 ó4   • [        U R                  5       5      $ r  )rn   rW  rÁ  s    rX   r½  Ú~CppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.get_call_ranges.<locals>.<lambda>
  s   € ¬¨Q¯^©^Ó-=Ô)>rW   rY  )r¢   r$   r"   r>  r`   r6  r£   )r+  rA  rÍ   r6  r7  rN  s         rX   Úget_call_rangesÚlCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.get_call_ranges  sa   € Ü! $¬Ô8JÐ(K×LÑLÐLÐLØ-1¯^©^Ó-=�Ü.1ØÑ>ñ/ç‘%ñ ,�Ñ+�Eô $ E›l¬U°?Ó-CÑC�Ø"Ð"rW   c              3   óf   >#   • U  H&  n[        T" U5      5      TR                  S -   :H  v •  M(     g7f)r5   N)rÓ   r8  )r*  rC  r  r+  s     €€rX   r,  ÚfCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.<genexpr>  s2   øé € ð â3�Eô ‘O EÓ*Ó+¨t×/KÑ/KÈaÑ/OÖOÚ3ùs   ƒ.1r5   r   c              3   ó\   >#   • U  H!  oR                   TR                  5       ;   v •  M#     g 7fr  )r+  r>  )r*  rÃ  r+  s     €rX   r,  r  &  s#   øé € ð ÚBX¸$Ÿ	™	 T§^¡^Ó%5Ö5ÒBXùs   ƒ),c                  óN  >^^• SmSn [        TR                  R                  R                  5       5       H  u  pTX-  -  mX-  n M     TR                  R	                  TR                  5       5      nU4S jmT" U5      =(       a     [        UU4S jTR                   5       5      $ )Nr   r5   c                 ó   >• U T:H  $ r  rV   )r·  Úcontiguous_index_exprs    €rX   Úis_contiguous_indexÚ–CppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.is_all_write_read_contiguous.<locals>.is_contiguous_index>  s   ø€ Ø'(Ð,AÑ'AÐ ArW   c              3   óØ   >#   • U  H_  n[        UR                  [        5      =(       a9    T" UR                  R                  R	                  TR                  5       5      5      v •  Ma     g 7fr  )r¢   r+  r$   r;  Úget_read_exprr.  )r*  rÃ  r  Úscheduler_buffers     €€rX   r,  ÚŒCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.is_all_write_read_contiguous.<locals>.<genexpr>A  s^   øé € ð Qò -C Dô !+¨4¯9©9´mÓ D÷ !"Ù$7Ø$(§I¡I§O¡O×$AÑ$AØ(8×(AÑ(AÓ(Có%&ó%"ô!"ò -Cùs   ƒA'A*)r5  r;  r™  r  Úget_write_exprr.  r4  rÇ  )r  r§   r^  Úwrite_index_exprr  r  r  r«  s       @@€€rX   Úis_all_write_read_contiguousÚyCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.is_all_write_read_contiguous0  s¬   ú€ Ø45Ð1Ø%&˜FÜ.6à .× 4Ñ 4× ?Ñ ?× EÑ EÓ Gö/¡
 ð !6¸¹Ñ EÐ 5Ø &¡¢ñ/ð 0>×/CÑ/C×/RÑ/RØ 0× 9Ñ 9Ó ;ó0Ð,õBñ $7Ð7GÓ#H÷ $ÌSõ Qð -=×,BÒ,BóQó Nð rW   rc  Nc                 ó�   >• U H?  nXR                   :X  d  M  [        U4S jTUR                      5       5      (       d  M=  Us  $    g )Nc              3   óÞ   >#   • U  Hb  nUR                   c  M  [        U4S j[        R                  R                  R
                  UR                      R                   5       5      v •  Md     g 7f)Nc              3   ó^   >#   • U  H"  nUR                   R                  5       T;   v •  M$     g 7fr  )r+  r.  )r*  rÃ  Úvisited_scheduler_nodess     €rX   r,  Ú�CppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.try_share_local_buffer.<locals>.<genexpr>.<genexpr>c  s/   øé € ð (&ò50¨Dð )-¯	©	×(:Ñ(:Ó(<Ð@WÖ(Wò50ùs   ƒ*-)rÇ   r4  r4   r¬  r3  Úname_to_bufrÇ  )r*  Úglobal_bufferr!  s     €rX   r,  Ú†CppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.try_share_local_buffer.<locals>.<genexpr>b  si   øé € ð S"ò:&¨ð (5×'9Ñ'9ó%&¤Cô (&ä45·G±G×4EÑ4E×4QÑ4QØ,9×,>Ñ,>ñ5*ç*/©%ð50ó(&÷ %&ð %&ò:&ùs   ƒA-˜AA-)rÁ  r4  rÇ   )Úlocal_buffer_layoutr+  Ú	local_bufÚlocal_to_global_buffersr!  s      €€rX   Útry_share_local_bufferÚsCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.try_share_local_buffer`  sR   ø€ Û-: 	Ø#6×:JÑ:JÕ#JÌsô S"ð :QØ(1¯©ò:&óS"÷ P"ó P"ð ,5Ò$4ñ .;ð $(rW   Úlocal_buffer_datarÍ   )rÇ   rÁ  F)Úlocal_buffer_numberT)2r¢   r#  Úclearr   r4  r6  r   r>  r$   r¦  r.  rW  rÓ   Úget_outputsrÇ  r+  r   rŒ  r-  r8  Úis_contiguousr  r5  ré  ÚFixedLayoutrr  rˆ   ÚBufferr  rÇ   rK   r›  Úadd_local_bufferr"   rq  rI  rj  r  r4   r¬  Úremover   Ú!cpp_outer_loop_fused_inner_countsÚCppOuterLoopFusedCountr+  rv  Úfinalize_kernelr‰  rŠ  Úfrom_iterable)r+  r+  r$  Úglobal_buffer_layoutÚsize_offsetr  Úlocal_buffer_strider  Úlocal_buffer_sizeÚszr&  r)  Úlocal_buf_prefixÚlocal_buffer_usedÚscoper/  rC  rh  Úremoved_bufferÚouter_fusion_cpp_kernel_proxyr  r(  r  r«  r!  rf  rn  Ú
nodes_listrA  s   `                   @@@@@€€€€rX   Ú$try_outer_loop_fusion_with_local_bufÚSCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_bufþ  sO  þ€ ô ˜dÔ$?×@Ñ@Ð@Ð@Ø!×'Ñ'Ô)Ø×ÑÔð#Ô&7ô #ð .0ˆMàBDÐ#Üõ à!×1Ñ1Ô3ó÷ ò ô <F»<Ð'Ø&*§n¡n×&6�Nä% n´m×DÑDÐDÐDØ+×/Ñ/°×0GÑ0GÓ0IÔJà&×3Ñ3×5Ñ5Ü˜~×9Ñ9Ó;Ó<ÀÓAá à'5×'AÑ'AÓ'CÀAÑ'FÐ$Üô ØBR×BXÒBXó÷ ó ð )9×(=Ñ(=˜Ü)¨-¼×9JÑ9J×KÑKÐKÐKØ/<×/GÑ/GÓ/IÐ,Ø&*×&BÑ&BÄSÙ+¨NÓ;óFñ '˜öð8 1×>Ñ>×@Ñ@Ù <× >Ñ >â$à9;Ð+Ø!5×!<Ñ!<¸RÑ!@˜Ù,;¸NÓ,KØ'˜Lð-Ð)ô #+Ð+<Ö"=˜BØ/×6Ñ6°q¸&ÔAØ"™LšFñ #>ô /1¯nªnØ0×7Ñ7Ø0×6Ñ6Ø-Ø/ó	/Ð+ö(ð" ,?Ð(á,BØ/ó-Ð)ö  1ä02·	²	Ø(8Ð'9¸¼3¸}Ó;MÐ:NÐ%OØ':ñ1Ð-ð *×0Ñ0Ð1BÔCØNPÐ3Ð4E×4JÑ4JÑKà/Ð0A×0FÑ0FÑG×NÑNØ)÷ñI '7ôP $ L×$5Ñ$5Ô6¸%Ü�}Ó%¨Ó)Û(5˜Ø+×0Ñ0Ñ<Ð<Ð<Ø×.Ñ.Ø(Ð*AÀ,×BSÑBSÑ*Töñ )6ð
 "×1Ñ1Ö3�EÜ% eÔ.@Ä-Ð-P×QÑQÐQÐQØ'+×'<Ñ'<¸\Ó'JÐ$Ø$×2Ñ2°5·?±?Ó3DÔEØ)×0Ñ0Ð1AÔBØ×%Ñ% e§o¡oÓ&7Ö8ñ 4ð ×>Ñ>Ø)¨4×+GÑ+G÷ñ ð +0×*?Ô*?˜ô Ÿ™×/Ñ/×6Ñ6°~ÖFñ +@ð !÷+ 7Ð6ô, ×9Ñ9×@Ñ@Ü×2Ò2ÜÐ1Ó2Ü,/°×0CÑ0CÓ,Dñôð 15×0OÑ0OØ)ó1Ð-ð ×,Ñ,Ø1Ø@”i—o‘o×3Ñ3°JÓ?Ð@ô÷? 7ðH ÷I 7Ô6ðH ús   ËD9RÐBRÒ
R,Fr  N)rn  r   r  r¢   r#  r-  r„   r  r   r  r6  r"   r$   r>  rq  rI  r6  )
rA  r+  r  rC  rC  Ú_nodesrh  rf  rn  rB  s
   `      @@@rX   Úcodegen_outer_loop_nodeÚ%CppScheduling.codegen_outer_loop_nodeî  s  û€ ð ×(Ñ(ˆÜ)0×)OÑ)OÐ&Ø=?ÐØ02ˆ
Ü˜$Ô ;×<Ñ<Ð<Ð<ði	Ô7R÷ i	ò i	ñV 4°D×9Ñ9à5SŒGÔ2Ø!×'Ñ'Ô)Ø×ÑÔô —‘×'Ñ'×-Ñ-¸eÐ-ÒDØ!×1Ñ1Ö3�EÜ% eÔ.@Ä-Ð-P×QÑQÐQÐQØ27·/±/Ó2C�FØ'+×'<Ñ'<¸\Ó'JÐ$Ø$×2Ñ2°6Ô:Ø ×0Ñ0Ð1AÖJñ 4÷ EÐDð :÷ EÕDús   Â,A8D.Ä.
D<c                 ó~  • U R                   n[        U[        5      (       a  U R                  U5        OTUR	                  5       nU R                  U5      nU R                  U5      nUR                  U5        UR                  XC5        U R                  5       nU[        R                  :”  a  U R                  S5        gg)z3
Turn an set of pre-fused nodes into a C++ kernel.
TN)rn  r¢   r#  rF  r>  r  rq  rI  r6  Ú_get_scheduled_num_argsrp  ÚMAX_FUSED_KERNEL_ARGS_NUMr~  )rA  r+  rn  rA  rh  Úargs_nums         rX   Úcodegen_nodeÚCppScheduling.codegen_node¹  s¤   € ð ×(Ñ(ˆä�dÔ7×8Ñ8Ø×(Ñ(¨Õ.à)-¯©Ó)9ˆEØ×'Ñ'¨Ó.ˆEØ#×4Ñ4°\ÓBÐØ×*Ñ*¨5Ô1Ø×(Ñ(Ð)9ÔAà×/Ñ/Ó1ˆØ”m×=Ñ=Ó=Ø×"Ñ" 4Õ(ð >rW   c                 óx   • [        U[        5      =(       a$    [        UR                  [        R                  5      $ r  )r¢   r$   r+  r   ÚCppTemplateBuffer)rA  r+  s     rX   Úis_cpp_templateÚCppScheduling.is_cpp_templateÏ  s,   € Ü˜$¤Ó.÷ 
´:Ø�I‰I”r×+Ñ+ó4
ð 	
rW   Útemplate_nodeÚepilogue_nodesÚprologue_nodesc                 ó`  • U(       a   eU Vs/ s H"  n[        U[        [        45      (       d  M   UPM$     nn[        S   S==   S-  ss'   [        S   S==   [	        U5      -  ss'   U R                  U5      (       d   S5       e[        [        U5      nUR                  u  nu  pVUS:X  d   e[        [        R                  UR                  5      nU Vs/ s H  oˆR                  PM     n	n[        S U	 5       5      (       d   S5       eS	 n
U
" XqR                  U	5      nUR                  UUU	S
9u  pÍU   [        UR                  5      (       d  UR                  5         U H  nUR                  5         M     U" 5       nSSS5        [         R"                  " U5         U/UQnU R%                  WUUR&                  5      nSSS5        [        UR                  5      (       aº  [	        UR(                  5      S:X  d   S5       eUR(                  S   R*                   H}  n[        UR                  [,        5      (       d   S5       e[        UR                  R                  [        R.                  5      (       d   S5       eUR                  R                  5         M     U R1                  WW5        UR3                  UU5        [         R4                  =R6                  UR6                  -  sl        U R9                  5         gs  snf s  snf ! , (       d  f       GN�= f! , (       d  f       GNa= f)z7
Codegen a CPP template, possibly with fused epilogues
ÚinductorÚcpp_templated_kernel_counterr5   Úcpp_epilogue_fusion_counterzlTemplate node passed to CppScheduler.codegen_template must be a SchedulerNode that wraps a CppTemplateBufferrV   c              3   óV   #   • U  H  n[        U[        R                  5      v •  M!     g 7fr  )r¢   r   rŒ  )r*  rp  s     rX   r,  Ú1CppScheduling.codegen_template.<locals>.<genexpr>ó  s"   é € ÐOÒ=N¸”:˜a¤×!2Ñ!2×3Ð3Ò=Nùr„  z9Epilogue nodes must all be instances of ir.ComputedBufferc                 ó¬   ^• T(       d  gU R                  5       U;   d   eXR                  5          R                  n[        U4S jU 5       5      (       + $ )NFc              3   ó˜   >#   • U  H?  n[        UR                  [        5      =(       a    UR                  R                  T;   v •  MA     g 7fr  )r¢   r+  r   )r*  rÃ  rS  s     €rX   r,  ÚZCppScheduling.codegen_template.<locals>.template_buffer_has_other_users.<locals>.<genexpr>ÿ  s@   øé € ð ò "�Dô ˜4Ÿ9™9Ô&7Ó8÷ 5Ø—I‘I—N‘N nÑ4ô5â!ùs   ƒAA
)r.  rÇ  r4  )Útemplate_bufferÚoutputs_by_namerS  rÇ  s     ` rX   Útemplate_buffer_has_other_usersÚGCppScheduling.codegen_template.<locals>.template_buffer_has_other_users÷  sZ   ø€ ö "Øà"×+Ñ+Ó-°Ó@Ð@Ð@Ø#×$<Ñ$<Ó$>Ñ?×EÑEˆEÜô ñ "óó ô ð rW   )Ú$flag_template_buffer_has_other_usersrS  NzSMulti outputs template should be with 1 output template buffer of MultiOutputLayoutr   z?Multi outputs template should be with ExternKernelSchedulerNodez7Multi outputs template has multi users with MultiOutput)r¢   r$   r"   r   rÓ   rP  r   r6  r   rO  r+  r4  r_  Úmake_kernel_renderr)   r>  r4   Úset_kernel_handlerÚdefine_kernelr›  ÚoutputsrÇ  r    rÃ  Úcodegen_commentÚcall_kernelr¬  r  Úfree_buffers_in_scheduler)rA  rR  rS  rT  Úepilogue_noderÍ   ÚrnumelÚctbrp  Úepilogue_ir_nodesr`  rb  rr  Úrenderr+  Úsrc_codeÚnode_scheduleÚkernel_namerÃ  s                      rX   Úcodegen_templateÚCppScheduling.codegen_templateÔ  sí  € ö "Ð!Ð!ñ
 "0ó
â!/�Ü˜-¬-Ô9KÐ)L×M÷ Ù!/ð 	ð 
ô 	�ÑÐ;Ó<ÀÑAÓ<Ü�ÑÐ:Ó;¼sÀ>Ó?RÑRÓ;Ø×#Ñ# M×2Ñ2ð 	
Øzó	
Ð2ô œ]¨MÓ:ˆØ&×,Ñ,‰ˆ‰;ˆAØ˜‹|Ðˆ|Ü$(¬×)=Ñ)=¸}×?QÑ?QÓ$Rˆá*ó;
Ú*�q�FŒF™Nð 	ð ;
ô ÑOÑ=NÓO×OÑOð 	
ØGó	
ÐOò	ñ 0OØ×.Ñ.Ð0Aó0
Ð,ð ×/Ñ/ØØ1UØ,ð 0ð 
‰ˆò
 Ü,¨]×-?Ñ-?×@Ñ@Ø×&Ñ&Ô(Û&�Ø—‘–ñ 'á“xˆH÷ ô ×!Ò! &Õ)Ø*Ð<¨^Ð<ˆMØ×,Ñ,¨X°}ÀfÇkÁkÓRˆK÷ *ô % ]×%7Ñ%7×8Ñ8ô �}×,Ñ,Ó-°Ó2ð ØeóÐ2ð &×-Ñ-¨aÑ0×6Ô6�Ü! $§)¡)Ô-F×GÑGð ØUóÐGô " $§)¡)§.¡.´"·.±.×AÑAð ØMóÐAð —	‘	×"Ñ"Ö$ñ 7ð 	×Ñ˜]¨KÔ8Ø×Ñ˜;¨Ô,Ü	�‰×Ò 6×#9Ñ#9Ñ9ÕØ×&Ñ&Õ(ùòU
ùò ;
÷: ŽVú÷ *Ö)ús)   ŽL±LÃLÄ8ALÆ!#LÌ
LÌ
L-c                 ó6   • U R                   R                  5       $ r  )rn  Úget_num_argsrG  s    rX   rI  Ú%CppScheduling._get_scheduled_num_args,  s   € Ø× Ñ ×-Ñ-Ó/Ð/rW   c                 ó   • U R                   $ r  r}  rG  s    rX   Úready_to_flushÚCppScheduling.ready_to_flush/  s   € Ø×#Ñ#Ð#rW   c                 ó   • g r  rV   rG  s    rX   Úcodegen_syncÚCppScheduling.codegen_sync2  s   € ØrW   c                 ó¤  • [         R                  R                  nXR                  ;   a  UR                  U   nU$ [        R
                  R                  (       a$  [        U[        R
                  R                  5      OSnSR                  SXdR                  5       /5      nXTR                  U'   [         R                  R                  (       a  UOSnUR                  [        [        R                  5      U5      nUR                  [        [        R                  5      U5      nUR                  SS5      nUR!                  S5      nUR#                  SU5      n	[$        (       a  UR#                  SU	S	-   5      n	XU	S	-     S
3n
['        5       nUc  U R(                  R*                  OUnUR-                  5       u    pÞ[         R                  R                  (       d  UR/                  SU< S35        UR1                  USS9  [         R                  R                  (       d  UR/                  S5        UR3                  UUR5                  5       SU
S9  U$ )NrS   rÍ   rð  rr  z#pragma CMTz//z
extern "C"r–   r5   z;
zasync_compile.cpp_pybinding(z, r'''T)Ústripz''')F)ÚgpuÚcpp_definition)r4   r¬  Úwrapper_codeÚsrc_to_kernelr   rð  Údescriptive_namesr'   r^  Únext_kernel_suffixÚcpp_wrapperr  rp   r,   ÚKERNEL_NAMEÚDESCRIPTIVE_NAMEÚrfindrù  rU   r=   rn  r›  Úcpp_argdefsrÂ   r\  re  Úgetvalue)rA  ro  rA  Úkernel_argsr  rq  Ú
fused_nameÚkernel_decl_nameÚ
first_charÚ	last_charÚkernel_definitionÚcompile_wrapperr›  rÍ   Ú	arg_typess                  rX   re  ÚCppScheduling.define_kernel5  sü  € Ü—'‘'×&Ñ&ˆØ×,Ñ,Ó,Ø!×/Ñ/°Ñ9ˆKðR ÐôK —:‘:×/×/ô & e¬V¯Z©Z×-IÑ-IÔJàð ð
 Ÿ(™( E¨:×7QÑ7QÓ7SÐ#TÓUˆKØ.9×!Ñ! (Ñ+Ü./¯g©g×.A×.A™{ÀxÐØ×'Ñ'¬¬K×,CÑ,CÓ(DÐFVÓWˆHØ×'Ñ'¬¬K×,HÑ,HÓ(IÈ;ÓWˆHð  ×'Ñ'¨°tÓ<ˆHð "Ÿ™¨Ó5ˆJØ Ÿ™ c¨:Ó6ˆIßŠ{à$ŸM™M¨#¨y¸1©}Ó=�	Ø#+¸ÀQ¹Ð#GÐ"HÈÐ LÐä,Ó.ˆOØ-8Ñ-@�4×$Ñ$×)Ò)ÀkˆDØ"×.Ñ.Ó0‰OˆAˆqÜ—7‘7×&×&Ø×)Ñ)Ø2°9±-¸vÐFôð ×"Ñ" 8°4Ð"Ñ8Ü—7‘7×&×&Ø×)Ñ)¨&Ô1Ø×!Ñ!ØØ×(Ñ(Ó*ØØ0ð	 "ñ ð ÐrW   c                 ó   • U R                   R                  5       nU(       GaK  U R                  XR                   R                  5      nU R	                  U R                   R                  U5        [
        R                  R                  (       af  [        R                  R                  R                  5         [        R                  R                  R                  UU R                   R                  5        U R                   R                  [        R                  R                  U5        [
        R                  R                  (       a(  [        R                  R                  R                  5         U R                  5         U R!                  S5        g rd  )rn  Úcodegen_groupre  Úscheduled_nodesrg  r   rð  Úenable_kernel_profiler4   r¬  r�  Ú write_kernel_context_guard_beginÚwrite_kernel_context_guardrh  Úwrite_kernel_context_guard_endrx  r~  )rA  ro  rq  s      rX   ÚflushÚCppScheduling.flushc  s  € Ø×$Ñ$×2Ñ2Ó4ˆßØ×,Ñ,Ø×+Ñ+×;Ñ;óˆKð × Ñ  ×!2Ñ!2×!BÑ!BÀKÔPÜ�z‰z×/×/Ü—‘×$Ñ$×EÑEÔGÜ—‘×$Ñ$×?Ñ?ØØ×%Ñ%×5Ñ5ôð ×Ñ×)Ñ)¬!¯'©'×*>Ñ*>ÀÔLÜ�z‰z×/×/Ü—‘×$Ñ$×CÑCÔEà×ÑÔ!Ø×Ñ˜uÕ%rW   c                 ór   • [         R                  R                  n[        UU5      nUR	                  X$5        g r  )r4   r¬  r�  r   Úwrite_provenance_debug_handle)rA  rp  rq  r  Údebug_handles        rX   rg  ÚCppScheduling.codegen_commentw  s1   € ä—'‘'×&Ñ&ˆÜ>Øàó
ˆð
 	×-Ñ-¨kÕHrW   )ry  rn  r  )7r  r  r  r  r  rq  r(  r   rJ  r   r6   ÚINPLACE_BUFFERSÚREDUCE_TO_SINGLE_ELEMENTrt  rx  r„   rr  ru  r@  rl   r~  rƒ  rx  r9  r   rh  r˜  r¦  r¹  r½  r   rÆ  r›  rš  r×  rÛ  r5  r$   r  r#  rF  r	   r"   rL  rP  r   rr  rI  rx  r{  re  r›  rg  r!  ry  rz  s   @rX   rp  rp  9  sŽ  ø‡ ð .<Ð�d˜>Ñ*Ó;ð !$ÐÙ!à×*Ñ*Ø×3Ñ3ð	
óÐð ð$¨%¯,©,ð $¸:ÀnÑ;Uó $ó ð$õ%ð&¨ô &òNò*òP=ðd¨xÐ8HÑ/Iô ò6òp>ò	<ð
Ø&ð
Ø/@ð
à	ô
ò4)òl

òò=ðC D¨Ñ$7ô CðJIKà)ôIKðV)àÐ/Ð1CÀ]ÐRÑSô)ð,
Ð$5ð 
¸$ô 
ð
V)à(ðV)ð !Ð!2Ñ3ðV)ð !Ð!2Ñ3ô	V)òp0ò$òô,ò\&÷(Iò IrW   rp  c                   óR   ^ • \ rS rSrU 4S jrS rS rS rS
S\4S jjr	S r
S	rU =r$ )r†  i‚  c                 ó  >• [         TU ]  5         [        5       U l        [	        5       U l        [        U R
                  5      U l        [        R                  " 5       U l
        U R                  R                  U R                  5        / U l        g r  )r?  r@  r?   r›  r7   Ú
loops_codeÚWorkSharingr¢  rÀ   rÁ   rÌ   rÃ   r–  )rA  rD  s    €rX   r@  ÚKernelGroup.__init__ƒ  s^   ø€ Ü‰ÑÔÜ“LˆŒ	Ü&›.ˆŒÜ˜dŸo™oÓ.ˆŒÜ×)Ò)Ó+ˆŒ
Ø�
‰
× Ñ  §¡Ô)Ø!ˆÕrW   c                 ó:   • U" U R                   [        5       /UQ76 $ r  )r›  r+   )rA  r7  r›  s      rX   r  ÚKernelGroup.new_kernelŒ  s   € Ù�4—9‘9Ô2Ó4Ð<°tÒ<Ð<rW   c                 ó€   • U =R                   U-  sl         U R                  nU R                  nUR                  X45        g r  )r–  r¥  r¢  r9  )rA  r  rA  r³   r¢  s        rX   r6  ÚKernelGroup.finalize_kernel�  s3   € Ø×Ò Ñ%ÕØ�‰ˆØ�W‰WˆØ× Ñ  Õ*rW   c                 óV   • U R                   R                  5       u  pn[        U5      nU$ r  )r›  r‰  rÓ   )rA  Úarg_defsÚ
_call_argsÚ
_arg_typesrK  s        rX   ru  ÚKernelGroup.get_num_args•  s'   € Ø+/¯9©9×+@Ñ+@Ó+BÑ(ˆ˜jÜ�x“=ˆØˆrW   r¸   c                 óš  • U R                   R                  5         U R                  (       d  g[        5       n[        R
                  R                  =(       a    [        R                  S;   nU(       a  UR                  S/5        UR                  S5        Uc  [        [        R                  5      OUnUc  [        [        R                  5      OUnU R                  R!                  5       u  n  nSR#                  S5      R%                  U5      n['        5       n[        R
                  R(                  (       a  SOSn	UR                  SU S	U	 S
U SU S3	5        UR+                  5          U(       aM  [,        R.                  R0                  n
U
b  S[        U
5      -   S-   OSnUR                  SXµ-    SXµ-    S3/5        U R                  R3                  5        H  u  pÍUR                  SU SU S35        M     UR5                  U R6                  5        S S S 5        UR9                  5       $ ! , (       d  f       UR9                  5       $ = f)NrS   )ÚlinuxrQ   z3#include <torch/csrc/inductor/aoti_runtime/utils.h>z+#include <torch/csrc/inductor/cpp_prefix.h>z,
é   ÚC10_ALWAYS_INLINE_ATTRIBUTEzextern "C" z void r˜   r¡  r–   Úgraph_rÍ   z9torch::aot_inductor::RAIIAtenRecordFunctionHandle record_z_("z", nullptr);rÕ   rº   r»   )rÌ   r  r–  r7   r   rð  r—  ÚsysÚplatformrÞ   rÂ   rp   r,   r†  r‡  r›  r‰  Úljustr^  rY   Úforce_inline_kernelrÄ   r4   r¬  Úgraph_idÚaliasesr\  r¥  rŠ  )rA  rÇ   r³   r—  r�  rq  r­  rÍ   Úfunc_export_declÚinline_attrrº  r  ÚoldÚnews                 rX   r•  ÚKernelGroup.codegen_groupš  sþ  € Ø�
‰
×ÑÔØ×#×#ØÜ‹~ˆô !'§
¡
× @Ñ @÷ !
ÄSÇ\Á\ð V
ñ F
Ðö !Ø�O‰OÐRÐSÔTØ�‰ÐDÔEð <@¹<œ3œ{×6Ñ6Ô7ÈTÐØ;?¹<”cœ+×6Ñ6Ô7ÈTˆØŸ™×.Ñ.Ó0‰ˆ�!�QØ—;‘;˜r“?×'Ñ'¨Ó1ˆÜ1Ó3Ðä-3¯Z©Z×-K×-KÑ)ÐQSð 	ð 	�‰ØÐ*Ð+¨6°+°¸aÐ@PÐ?QÐQRÐS[ÐR\Ð\]Ð^ô	
ð
 �[‰[�]Þ$ÜŸ7™7×+Ñ+�Ø;CÑ;O˜¤C¨£MÑ1°CÒ7ÐUW�Ø—‘ð&Ø&,Ñ&:Ð%;¸3¸vÑ?SÐ>TÐT`ðbðôð !ŸI™I×-Ñ-Ö/‘�Ø—‘  s e¨3¨s¨e°1Ð5Ö6ñ 0à�K‰K˜Ÿ™Ô(÷ ð �}‰}‹Ð÷ Œ]ð �}‰}‹Ðús   Å*B+H-È-
I
c                 ób   • U R                   R                  5       u  p4nUR                  UUSUS9  g )NF)Útritonr’  )r›  r‰  Úgenerate_kernel_call)rA  r  rq  rÍ   Ú	call_argsr’  s         rX   rh  ÚKernelGroup.call_kernelÈ  s8   € Ø"&§)¡)×"7Ñ"7Ó"9Ñˆ�iØ×$Ñ$ØØØØð	 	%ò 	
rW   )r›  r¥  r–  rÌ   r¢  r  )r  r  r  r  r@  r  r6  ru  rp   r•  rh  r!  ry  rz  s   @rX   r†  r†  ‚  s.   ø† õ"ò=ò+òñ
,¨#õ ,÷\
ð 
rW   r†  c                   ó8   • \ rS rSrS rS rS rS rS rS r	Sr
g	)
r¦  iÒ  c                 ó`   • Xl         SU l        S U l        [        R                  " 5       U l        g rd  )r³   Úin_parallelra  rÀ   rÁ   rÌ   )rA  r³   s     rX   r@  ÚWorkSharing.__init__Ó  s'   € ØŒ	Ø ˆÔØˆÔÜ×)Ò)Ó+ˆ�
rW   c                 óò  • U R                   (       a  XR                  :w  a  U R                  5         U R                   (       d¶  Xl        SU l         [        R                  R
                  (       a  U R                  R                  S5        OU R                  R                  SU S35        U R                  R                  U R                  R                  5       5        U R                  R                  S5        g g )NTz#pragma omp parallelz!#pragma omp parallel num_threads(r–   zint tid = omp_get_thread_num();)rÈ  ra  r  r   rð  ri  r³   rÂ   rÌ   rÃ   rÄ   )rA  r  s     rX   r  ÚWorkSharing.parallelÙ  s¬   € Ø×× ×+;Ñ+;Ó ;à�J‰JŒLØ××Ø&ÔØ#ˆDÔÜ�z‰z×)×)Ø—	‘	×#Ñ#Ð$:Õ;à—	‘	×#Ñ#Ð&GÈÀyÐPQÐ$RÔSØ�J‰J×$Ñ$ T§Y¡Y×%5Ñ%5Ó%7Ô8Ø�I‰I×ÑØ1õð  rW   c                 ór   • U R                   (       a  U R                  R                  S5        U R                   $ )Nz#pragma omp single)rÈ  r³   rÂ   rG  s    rX   r)  ÚWorkSharing.singleé  s*   € Ø××Ø�I‰I×ÑÐ 4Ô5Ø×ÑÐrW   c                 óF   • U R                   R                  5         SU l        g rd  )rÌ   r  rÈ  rG  s    rX   r  ÚWorkSharing.closeî  s   € Ø�
‰
×ÑÔØ ˆÕrW   c                 ó:   • U R                   R                  5         U $ r  )rÌ   r‡  rG  s    rX   r‡  ÚWorkSharing.__enter__ò  s   € Ø�
‰
×ÑÔØˆrW   c                 ó<   • U R                   R                  XU5        g r  )rÌ   rŽ  rŠ  s       rX   rŽ  ÚWorkSharing.__exit__ö  s   € Ø�
‰
×Ñ˜H¨vÕ6rW   )r³   rÈ  ra  rÌ   N)r  r  r  r  r@  r  r)  r  r‡  rŽ  r!  rV   rW   rX   r¦  r¦  Ò  s    † ò,òò  ò
!òõ7rW   r¦  c                   óœ  • \ rS rSr% Sr\\R                     \S'   Sr	\\R                     \S'   \R                  R                  r\R                  \S'   \R                  R                  r\R                  \S'   \R                  R                  r\R                  \S'   Sr\\S	'   S
r\\S'   S
r\\S'   S
r\\S'   S
r\\S'   S rS rS rSrg)rV  iú  Nr§   rO  rP  r   rQ  r   r  FÚsimd_ompÚsimd_vecÚ	collapsedrW  c                 óv   • [         R                  " 5       nU(       a  UR                  5       U l        g SU l        g rm  )r   rk  rj  Úsimd_nelements)rA  r£  s     rX   Ú__post_init__ÚLoopLevel.__post_init__  s-   € ô .9×-EÒ-EÓ-GˆÞAO >×#;Ñ#;Ó#=ˆÕÐUVˆÕrW   c                 ó  • [         R                  " U5      n[        U R                  U R                  5      nX#l        SUl        [        UR                  U5      U-  Ul        U R                  Ul	        SUl
        U R                  Ul        U$ )NTF)rõ   re  rV  r§   rO  rQ  rÖ  r   r   r  r×  rW  )rA  r   Úsympy_factorr  s       rX   r  ÚLoopLevel.tile  sl   € Ü—}’} VÓ,ˆÜ˜Ÿ™ 4§9¡9Ó-ˆØ!Œ
ØˆŒÜ" 4§9¡9¨lÓ;¸lÑJˆŒØŸ™ˆŒØˆŒØ ×-Ñ-ˆÔØˆrW   c                 ó  • [        U R                  5      n[        U R                  5      n[        R                  R
                  (       a  X:X  a  g U R                  (       a   U R                  S:”  a  SU R                   S3OSnU R                  (       aL  SnU R                  S:”  a  USU R                   S3-  nU R                  (       a  UR                  SSU 35      nO[U R                  (       a  SnOGU R                  (       a  S	U 3nO0U R                  (       d  [        R                  " 5       (       a  S
nOSn[         SU R                   SU 3nU R                   SU 3nU R                   R"                  (       a%  U R                   S[        U R                   5       3nO;U R                   S[        U R                   5       S[        U R                   5       S3nSU SU SU S3nU R$                  (       d  U(       d  U/$ XH/$ )Nr5   zsimd simdlen(z) rS   z#pragma omp forz
 collapse(r–   z for z#pragma omp z#pragma GCC ivdepr˜   r¨  r×  r§  z+=(z == 0 ? 1 : zfor(r½   )rE   rP  rO  r   rð  Úno_redundant_loopsrÕ  rÙ  r  r  rÖ  rW  r   r©  rJ   r§   rQ  r  r×  )	rA  Úoffset_exprÚ	size_exprÚsimdÚline1Ú
offset_strr¥  Ú	steps_strÚline2s	            rX   r!  ÚLoopLevel.lines&  s½  € Ü! $§+¡+Ó.ˆÜ §	¡	Ó*ˆ	Ü�:‰:×(×(¨[Ó-EØð �}�} ×!4Ñ!4°qÓ!8ð ˜D×/Ñ/Ð0°Ñ3àð 	ð
 �=�=à%ˆEØ�}‰}˜qÓ Ø˜: d§m¡m _°AÐ6Ñ6�Ø�}�}ØŸ™ g°°t°f¨~Ó>�øØ�]�]Ø‰EØ�]�]Ø" 4 &Ð)‰EØ×"×"¤{×'9Ò'9×';Ñ';Ø'‰EàˆEÜ"�| 1 T§X¡X J¨a°¨}Ð=ˆ
Ø—h‘h�Z˜q  Ð,ˆØ�:‰:××ØŸ8™8˜* B¤{°4·:±:Ó'>Ð&?Ð@‰Ið
 —8‘8�*˜C¤¨D¯J©JÓ 7Ð8ð 9Ü" 4§:¡:Ó.Ð/¨qð2ð ð �z�l " X J¨b°°¸1Ð=ˆØ�>�>¦Ø�7ˆNØˆ~ÐrW   )rÙ  )r  r  r  r  r§   r   rõ   rý  r   rO  rö   r÷   rP  r   ÚOnerQ  r  rn   rÕ  rl   rÖ  r×  rW  rÚ  r  r!  r!  rV   rW   rX   rV  rV  ú  s®   ‡ à $€Cˆ�%—*‘*Ñ	Ó$Ø!%€Dˆ(�5—:‘:Ñ
Ó%ØŸ™Ÿ™€FˆE�J‰JÓ%ð #ŸW™WŸ\™\€J�—
‘
Ó)ØŸ™Ÿ™€Eˆ5�:‰:Ó#Ø€HˆcÓØ€HˆdÓØ€HˆdÓØ€IˆtÓØ€L�$Óò
Wò	õ'rW   rV  c                   ó¤   • \ rS rSr% SrSr\\\      \	S'   Sr
\\   \	S'   \S\4S j5       rS r\S 5       rS	 rS
 rS\4S jrS rS\4S jrSrg)r]  iP  a>  
A loop-nest-like structure. It is built with the `build` method
as a loop nest and then will perform loop-tiling at some depth.

A typical case is for vectorization, where we typically do loop-tiling
at the innermost loop level. A more complicated case is when we do
2D tiling at both the innermost and outer levels.
NrX  rr  c                 ó0  • U R                   nU R                  nU R                  nUc   eSn[        [	        X5      5       HG  u  nu  pg[        Xg5      nU(       d  U/nOUR                  U5        XS:¼  d  M6  U R                  Ul        MI     [        U5      n	U	$ )z4Build a LoopNest with the given `kernel` as the leafN)	r  rd  rO  rÅ   r]  rV  r  rW  r]  )
rr  r  rd  rO  rX  Úloop_idxr§   rO  r  r_  s
             rX   r8  ÚLoopNest.build^  s“   € ð —?‘?ˆØ—‘ˆØ ×0Ñ0ˆØÑ*Ð*Ð*à+/ˆÜ%.¬s°8Ó/DÖ%EÑ!ˆH‘k�sÜ˜SÓ'ˆDÞØ˜‘à—‘˜TÔ"ØÕ*Ø$*×$7Ñ$7�Ö!ñ &Fô ˜U“Oˆ	ØÐrW   c                 ó,   • [        U R                  5      $ r  )rl   rX  rG  s    rX   Ú__bool__ÚLoopNest.__bool__s  s   € Ü�D—J‘JÓÐrW   c                 ó  • U R                   c
  [        SSS9$ SnSnU R                   S   R                  n[        R                  " S5      nU R                    H=  nUR                  U:w  a    O,U[        UR                  UR                  5      -  nUS-  nM?     S nU" U R                   5      nS[        4S jnU[        U R                   5      :  Ga1  [        U[        R                  5      (       Ga  [        U R                   U   R                  [        R                  5      (       aÛ  US-  [        U R                   U   R                  U R                   U   R                  5      :  aš  Ub0  X':”  a+  U R                   U   R                  (       a  U" U 5      (       dg  UnSnU R                   U   R                  n[        U[        U R                   5      5       H'  n	U R                   U	   R                  U:w  a    O	US-  nM)     [        X!S9$ )aY  
Maximal allowed depth for parallelism: All reduction or non-reduction levels.
When the range of the first inner loop beyond the maximum parallel depth is much
larger than the range of all outer loops within the maximum parallel depth,
change the starting depth of parallelism to the first inner loop and recalculate
the maximum parallel depth.
r   rB  r5   c                 óV   • [        U 5       H  u  pUR                  (       d  M  Us  $    g r  )rÅ   rÖ  )rX  ræ   r  s      rX   Úget_simd_vec_depthÚ7LoopNest.max_parallel_depth.<locals>.get_simd_vec_depthŒ  s&   € ä$ UÖ+‘�Ø—=—=‘=Ø’Hñ ,ð rW   r_  c                 ó�   • [        U R                  [        5      (       d   e[        S U R                  R                   5       5      $ )Nc              3   óL   #   • U  H  n[        U[        5      (       + v •  M     g 7fr  )r¢   r  )r*  rr  s     rX   r,  ÚILoopNest.max_parallel_depth.<locals>.has_scalar_kernel.<locals>.<genexpr>—  s$   é € ð â6�Fô ˜v¤|Ó4×4Ñ4Ú6ùrÏ  )r¢   rr  r  rg   r¤  )r_  s    rX   Úhas_scalar_kernelÚ6LoopNest.max_parallel_depth.<locals>.has_scalar_kernel•  sA   € Ü˜i×.Ñ.´×?Ñ?Ð?Ð?Üñ à'×.Ñ.×6Ò6óó ð rW   r\  )rX  r  rW  rõ   re  r   rO  rQ  r]  rÓ   r¢   r^  )
rA  r  Ú	max_depthrW  Ú	num_stepsr  ró  Úsimd_vec_depthrø  ræ   s
             rX   r'  ÚLoopNest.max_parallel_depthv  s³  € ð �:‰:ÑÜ °¸qÑAÐAàˆØˆ	Ø—z‘z !‘}×1Ñ1ˆÜ—M’M !Ó$ˆ	Ø—J”JˆDØ× Ñ  LÓ0ÙØ!¤H¨T¯Y©Y¸¿
¹
Ó$CÑCˆIØ˜‰NŠIñ	 ò	ñ ,¨D¯J©JÓ7ˆð	¬ô 	ð œ˜DŸJ™J›Ô'Ü˜9¤e§m¡m×4Ò4Ü˜4Ÿ:™: iÑ0×5Ñ5´u·}±}×EÑEØ˜C‘Ü�t—z‘z )Ñ,×1Ñ1°4·:±:¸iÑ3H×3NÑ3NÓOóPð Ñ*ØÓ.Ø—J‘J˜yÑ)×6×6Ù% d×+Ñ+ð $ˆKØˆIØŸ:™: kÑ2×?Ñ?ˆLÜ˜;¬¨D¯J©J«Ö8�Ø—:‘:˜a‘=×-Ñ-°Ó=ÙØ˜Q‘’	ñ 9ô ¨IÑOÐOrW   c                 óî  • UR                   U R                  5       R                   ::  d   S5       eU R                  c   e[        U R                  5      UR                   :¼  d   eU R                  UR                     nUR                   Ul        UR                  (       a  [        =R                  S-  sl        [        UR                  S-   UR                   5       H  nSU R                  U   l
        M     g )Nz?Parallel depth cannot exceed the maximal allowed parallel depthr5   T)r  r'  rX  rÓ   r  r  rW  r   Úparallel_reduction_countr^  r×  )rA  r  r  ræ   s       rX   r(  ÚLoopNest.mark_parallelµ  sË   € Ø×'Ñ'¨4×+BÑ+BÓ+D×+SÑ+SÓSð 	
ØMó	
ÐSð �z‰zÑ%Ð%Ð%Ü�4—:‘:‹ )×":Ñ":Ó:Ð:Ð:Ø�z‰z˜)×/Ñ/Ñ0ˆØ!×0Ñ0ˆŒØ××ä×,Ò,°Ñ1Õ,Ü�y×,Ñ,¨qÑ0°)×2JÑ2JÖKˆAØ&*ˆD�J‰J�q‰MÖ#ò LrW   c                 óœ   • U R                   (       d   eU R                   U   R                  U5      U R                   U'   U R                   U   $ )z­
Do loop-tiling at the `depth` level with `factor`.
    for (x0 = 0; x0 < x0_end; x0++)
    ->
    for (x0 = 0; x0 < x0_end; x0 += factor)
See details in Note [tiled_size].
)rX  r  )rA  r  r   s      rX   r  ÚLoopNest.tileÃ  sA   € ð �z�zÐˆzØ ŸJ™J uÑ-×2Ñ2°6Ó:ˆ�
‰
�5ÑØ�z‰z˜%Ñ Ð rW   r¸   c                 ó@   • U R                   (       d   eU R                   $ r  ©rr  rG  s    rX   r
  ÚLoopNest.get_kernelÏ  s   € Ø�{�{Ðˆ{Ø�{‰{ÐrW   c                 ó   • Xl         g r  r	  rN  s     rX   r  ÚLoopNest.set_kernelÓ  s   € Ø�rW   Úlevelc                 óÜ   • U R                   (       d   e[        U R                   5      U:¼  d   eU[        U R                   5      :X  a  S OU R                   US  n[        X R                  5      $ r  )rX  rÓ   r]  rr  )rA  r	  rX  s      rX   rp  ÚLoopNest.from_loop_levelÖ  sT   € Ø�z�zÐˆzÜ�4—:‘:‹ %Ó'Ð'Ð'Ø¤ T§Z¡Z£Ó0‘°d·j±jÀÀÐ6HˆÜ˜Ÿ{™{Ó+Ð+rW   r	  )r  r  r  r  r  rX  r   r5  rV  r   rr  rF  rü  r8  rï  r%   r'  r(  r  r
  r  rn   rp  r!  rV   rW   rX   r]  r]  P  s‡   ‡ ñð (,€Eˆ8�D˜‘OÑ$Ó+Ø"&€FˆH�YÑÓ&àð�ió ó ðò( ð ñ<Pó ð<Pò|+ò
!ð˜Iô òð, S÷ ,rW   r]  )NNNr  )­rÀ   Údataclassesr`  r‰  ry  rb  rÉ   r¶  rD  Úcollections.abcr   r   Úenumr   Útypingr   r   r   r	   rõ   r„   Útorch.fxÚtorch._inductorr
   Útorch._prims_commonr   r   Útorch.utils._ordered_setr   Útorch.utils._sympy.functionsr   r   r   Útorch.utils._sympy.symbolr   r   r   Ú_dynamo.utilsr   rS   r   r   r   r   r   Údebugr   rQ  r   r3  r   r   r    r!   r"   r#   r$   Úutilsr%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   Úvirtualizedr1   r2   r3   r4   Úcommonr6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   Ú	cpp_utilsrB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   r·  rU   ÚcacherY   Ú_loggingÚgetArtifactLoggerr  Úschedule_logÚNATIVE_OMP_RTYPESÚRTYPE_TO_CPPrÇ  ÚPYTHON_TO_CPPÚCONTAINER_PYTHON_TO_CPPrö  r÷  rƒ   rõ  ro   rl   rø  rù  r[  r  rú  rû  rw   r5  rˆ   r   r�   r‘   r  r°   r¤   rý  rp   rÐ   rn   râ   rç   rñ   Ú	lru_cacherû   r  r  Ú	dataclassr  r#  r|  r�  rŸ  Ú_initialize_pointwise_overridesr   r<  r@  rF  r  rB  r£   rJ  rL  r  ro  rh  rp  r†  r¦  rV  r]  rV   rW   rX   Ú<module>r&	     s  ðä Û Û Û Û Û Û 	Û 
Û ß .Ý ß -Ó -ã ã Û Ý (ß @Ý /ß KÑ Kß OÑ Oå %ß <Õ <Ý ;Ý  ÷÷ ñ ÷÷ ÷ ó ÷ >Ó =÷÷ ÷ ó ÷÷ ÷ ÷ ñ ð& �l‰l˜gÑ%€ð ‡�ñ:ó ð:ð �~‰~×/Ñ/°¸*ÓE€áÒBÓCÐ àØØØØØØØØØ ñ€ñ !òóÐ ð  ØØØØØ#Ø&ØØØñ€ð ØñÐ ð 
‡N�NØ	‡M�Mð€ð 
‡M�MØ	‡K�KØ	‡N�NØ	‡M�MØ	‡J�JØ	‡K�KØ	‡J�JØ	‡K�KØ	‡K�KØ	×ÑØ	×Òð*Ð �T˜%Ÿ+š+Ñ&ó ò)òDð Ø$(Øñ2)ð
 �E—L’LÑ!õ2)òjð-Ø
ð-à�jŠjð-ð ð-ð —
’
ð	-ð
 �jŠjð-ð ô-ð`Ø�3˜Ð#Ñ$ðàðð ðð �;Š;ð	ð
 
ˆs�Cˆx‰ôðBV˜^ð V°3ð VÀ#ô Vð/¨ô /ð2 ×Òð-�U—Z’Zð - e§l¢ló -ó ð-ð ×Òð; u§z¢zð ;¸¿
º
ð ;ÐPSó ;ó ð;ð| ×ÒàFJñ!Ø�:Š:ð!Ø!ŸLšLð!Ø6>¸s±mô!ó ð!ð ×Ò÷ð ó ðô^!Ð"4ô ^!÷B!ñ !òB&ôx"�;ô x"ðv × ,Ò ,¨UÔ 3ô|7�lô |7ð~ × /Ò /°Ô 9Ø × %Ò %Ô 'ô7˜ô 7ôX�ô Xôvb�9ô bôJ[
�lô [
ð|)$ ð )$¨e°H¸U¿[º[Ñ4IÈ4Ð4OÑ.Pô )$÷X_Kñ _KôDg	
�Yô g	
ôT 
˜9ô  
ôFF�tô FôFI�Nô FI÷RM
ñ M
÷`%7ñ %7ðP ×Ò÷Rð Ró ðRðj ×Ò÷I,ð I,ó ñI,rW   