ó
    Eñi„ü  ã                  ót  • S SK Jr  S SKrS SKrS SKrS SKrS SKrS SKJr  S SK	J
r
  S SKJrJrJrJrJr  S SKrS SKrS SKrSSKJr  SSKJr  SS	KJrJr  SS
KJrJr  SSKJr  SSK J!r!J"r"  SSK#J$r$  SSK"J%r%  SSK&J'r'J(r(  SSK)J*r*  SSK+J,r,J-r-  SSK.J/r/J0r0J1r1J2r2J3r3  SSK4J5r6J7r7  SSK8J9r9J:r:J;r;J<r<J=r=J>r>J?r?J@r@JArA  SSKBJCrC  SSKDJErE  SSKFJGrGJHrHJIrI  \(       a  S SKJJKrKJLrL  SSK)JMrMJNrN  SSKOJPrP  \R¢                  " \R5      rSS rT " S S\U5      rV " S  S!\?5      rW\W" 5       R°                  rY\?" 5       R°                  rZ\R¶                  S"\R¸                  S#\Rº                  S$\R¼                  S%\R¾                  S&\RÀ                  S'\RÂ                  S(\RÄ                  S)\RÆ                  S*\RÈ                  S+\RÊ                  S,\RÌ                  S-\RÎ                  S.0rhS/ riS0 rj " S1 S2\>5      rk\kRÙ                  S35         " S4 S5\:5      rm\RÜ                   " S6 S75      5       roS8 rpS9 rq " S: S;\H5      rr " S< S=\I5      rsg)>é    )ÚannotationsN)Údefaultdict)Úinf)ÚAnyÚcastÚOptionalÚTYPE_CHECKINGÚUnioné   )Úis_integer_dtype)Ú
OrderedSet)ÚFloorDivÚModularIndexing)Úsymbol_is_typeÚSymT)ÚValueRangesé   )ÚconfigÚir)ÚHalideCodeCache)Úget_reduction_combine_fn)Úis_metric_table_enabledÚlog_kernel_metadata)ÚAddParenHandler)ÚHalideInputSpecÚ
HalideMeta)Úget_bounds_index_exprÚget_kernel_metadataÚparallel_num_threadsÚsympy_index_symbolÚ
sympy_subs)Ú_opsÚVé   )	ÚBackendFeatureÚCSEVariableÚDeferredLineÚIndentedBufferÚKernelArgTypeÚOpOverridesÚPythonPrinterÚSizeArgÚ	TensorArg)ÚDTYPE_TO_CPP)Úcexpr)Úconstant_reprÚ
SIMDKernelÚSIMDScheduling)ÚCallableÚSequence)ÚReductionTypeÚ	StoreMode)ÚBlockShapeTypec                ó>  • [        U [        5      (       aZ  SU s=::  a  S::  dM  O  [        R                  " [        R                  5      nXR
                  :X  a  gXR                  :X  a  gSU < S3$ [        U [        5      (       a  S[        U 5       S3$ [        U 5      $ )Ni   €iÿÿÿzhl.Int(64).min()zhl.Int(64).max()zhl.i64(Ú)zhl.f64()
Ú
isinstanceÚintÚtorchÚiinfoÚint64ÚminÚmaxÚfloatr0   Úrepr)ÚvalÚinfos     Ú[/home/mande/repo/quber/.venv/lib/python3.13/site-packages/torch/_inductor/codegen/halide.pyÚhalide_constantrF   >   s€   € Ü�#”s×Ñ [°CÕ%E¸:Õ%EÜ�{Š{œ5Ÿ;™;Ó'ˆØ—(‘(‹?Ø%Ø—(‘(‹?Ø%Ø˜™˜qÐ!Ð!Ü�#”u×ÑØœ sÓ+Ð,¨AÐ.Ð.Ü�‹9Ðó    c                  ó,   ^ • \ rS rSrSU 4S jjrSrU =r$ )ÚUnsupportedéK   c                ó*   >• [         TU ]  SU 35        g )Nz!halide backend does not support: )ÚsuperÚ__init__)ÚselfÚthingÚ	__class__s     €rE   rM   ÚUnsupported.__init__L   s   ø€ Ü‰ÑÐ<¸U¸GÐDÕErG   © ©ÚreturnÚNone)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__rM   Ú__static_attributes__Ú__classcell__©rP   s   @rE   rI   rI   K   s   ø† ÷Fõ FrG   rI   c                  óÞ   ^ • \ rS rSr\S 5       r\S 5       rS rS rS r	\	r
S r\rS rS	 rS
 rS rS rS rS rS rS rS rS rS rS rS rS rS rU 4S jrS r\rS r S r!Sr"U =r#$ )ÚHalidePrinteréP   c                óD   • S[         R                  R                   SU  S3$ )Núhl.cast(ú, r9   )r#   ÚkernelÚindex_dtype©Úexprs    rE   Ú
cast_indexÚHalidePrinter.cast_indexQ   s"   € àœ!Ÿ(™(×.Ñ.Ð/¨r°$°°qÐ9Ð9rG   c                ó   • SU  S3$ )Núhl.cast(hl.Float(32), r9   rR   re   s    rE   Ú
cast_floatÚHalidePrinter.cast_floatU   s   € à'¨ v¨QÐ/Ð/rG   c                ó   • SU S3$ )Núhl.f32(r9   rR   ©rN   rf   s     rE   Ú_print_FloatÚHalidePrinter._print_FloatY   s   € Ø˜˜˜aÐ Ð rG   c                ó|   • [        UR                  5      S:X  d   eSU R                  UR                  S   5       S3$ )Nr$   rn   r   r9   ©ÚlenÚargsÚ_printro   s     rE   Ú_print_ToFloatÚHalidePrinter._print_ToFloat\   ó9   € Ü�4—9‘9‹~ Ó"Ð"Ð"à˜Ÿ™ T§Y¡Y¨q¡\Ó2Ð3°1Ð5Ð5rG   c                óš   • [        UR                  5      S:X  d   eU R                  SU R                  UR                  S   5       S35      $ )Nr$   ú	hl.floor(r   r9   ©rt   ru   rg   rv   ro   s     rE   Ú_print_floorÚHalidePrinter._print_floora   óB   € Ü�4—9‘9‹~ Ó"Ð"Ð"à�‰ ¨4¯;©;°t·y±yÀ±|Ó+DÐ*EÀQÐGÓHÐHrG   c                óš   • [        UR                  5      S:X  d   eU R                  SU R                  UR                  S   5       S35      $ )Nr$   ú	hl.trunc(r   r9   r|   ro   s     rE   Ú_print_TruncÚHalidePrinter._print_Trunch   r   rG   c                óš   • [        UR                  5      S:X  d   eU R                  SU R                  UR                  S   5       S35      $ )Nr$   úhl.ceil(r   r9   r|   ro   s     rE   Ú_print_ceilingÚHalidePrinter._print_ceilingo   sB   € Ü�4—9‘9‹~ Ó"Ð"Ð"à�‰ ¨$¯+©+°d·i±iÀ±lÓ*CÐ)DÀAÐFÓGÐGrG   c                óJ   • SU R                  U R                  U5      5       S3$ ©Nzhl.sqrt(r9   )rk   rv   ro   s     rE   Ú_helper_sqrtÚHalidePrinter._helper_sqrtt   s$   € à˜$Ÿ/™/¨$¯+©+°dÓ*;Ó<Ð=¸QÐ?Ð?rG   c                óÎ   • U R                  UR                  S   5      nU R                  UR                  S   5      nU R                  UR                  S   5      nSU SU SU S3$ )Nr   r$   r   ú
hl.select(rb   r9   )Údoprintru   )rN   rf   ÚcÚpÚqs        rE   Ú_print_WhereÚHalidePrinter._print_Wherex   s_   € Ø�L‰L˜Ÿ™ 1™Ó&ˆØ�L‰L˜Ÿ™ 1™Ó&ˆØ�L‰L˜Ÿ™ 1™Ó&ˆØ˜A˜3˜b   2 a S¨Ð*Ð*rG   c                ón  • [        UR                  5      S:X  a  U R                  UR                  S   5      $ [        UR                  5      S-  nU R                  [        R                  " UR                  S U 6 5      nU R                  [        R                  " UR                  US  6 5      nSU SU S3$ )Nr$   r   r   úhl.min(rb   r9   )rt   ru   rv   ÚsympyÚMin©rN   rf   ÚmidÚaÚbs        rE   Ú
_print_MinÚHalidePrinter._print_Min~   s’   € Üˆt�y‰y‹>˜QÓà—;‘;˜tŸy™y¨™|Ó,Ð,ä�$—)‘)‹n Ñ!ˆà�K‰KœŸ	š	 4§9¡9¨T¨c ?Ð3Ó4ˆà�K‰KœŸ	š	 4§9¡9¨S¨T ?Ð3Ó4ˆØ˜˜˜2˜a˜S Ð"Ð"rG   c                ón  • [        UR                  5      S:X  a  U R                  UR                  S   5      $ [        UR                  5      S-  nU R                  [        R                  " UR                  S U 6 5      nU R                  [        R                  " UR                  US  6 5      nSU SU S3$ )Nr$   r   r   úhl.max(rb   r9   )rt   ru   rv   r–   ÚMaxr˜   s        rE   Ú
_print_MaxÚHalidePrinter._print_MaxŠ   s’   € Üˆt�y‰y‹>˜QÓà—;‘;˜tŸy™y¨™|Ó,Ð,ä�$—)‘)‹n Ñ!ˆà�K‰KœŸ	š	 4§9¡9¨T¨c ?Ð3Ó4ˆà�K‰KœŸ	š	 4§9¡9¨S¨T ?Ð3Ó4ˆà˜˜˜2˜a˜S Ð"Ð"rG   c                óš   • [        UR                  5      S:X  d   eU R                  SU R                  UR                  S   5       S35      $ )Nr$   úhl.abs(r   r9   r|   ro   s     rE   Ú
_print_AbsÚHalidePrinter._print_Abs—   sB   € Ü�4—9‘9‹~ Ó"Ð"Ð"à�‰ ¨¯©°T·Y±Y¸q±\Ó)BÐ(CÀ1ÐEÓFÐFrG   c                ó|   • [        UR                  5      S:X  d   eSU R                  UR                  S   5       S3$ )Nr$   úhl.cos(r   r9   rs   ro   s     rE   Ú_print_OpaqueUnaryFn_cosÚ&HalidePrinter._print_OpaqueUnaryFn_cosœ   ry   rG   c                ó|   • [        UR                  5      S:X  d   eSU R                  UR                  S   5       S3$ )Nr$   úhl.cosh(r   r9   rs   ro   s     rE   Ú_print_OpaqueUnaryFn_coshÚ'HalidePrinter._print_OpaqueUnaryFn_cosh¡   ó9   € Ü�4—9‘9‹~ Ó"Ð"Ð"à˜$Ÿ+™+ d§i¡i°¡lÓ3Ð4°AÐ6Ð6rG   c                ó|   • [        UR                  5      S:X  d   eSU R                  UR                  S   5       S3$ )Nr$   úhl.acos(r   r9   rs   ro   s     rE   Ú_print_OpaqueUnaryFn_acosÚ'HalidePrinter._print_OpaqueUnaryFn_acos¦   r¯   rG   c                ó|   • [        UR                  5      S:X  d   eSU R                  UR                  S   5       S3$ )Nr$   úhl.sin(r   r9   rs   ro   s     rE   Ú_print_OpaqueUnaryFn_sinÚ&HalidePrinter._print_OpaqueUnaryFn_sin«   ry   rG   c                ó|   • [        UR                  5      S:X  d   eSU R                  UR                  S   5       S3$ )Nr$   úhl.sinh(r   r9   rs   ro   s     rE   Ú_print_OpaqueUnaryFn_sinhÚ'HalidePrinter._print_OpaqueUnaryFn_sinh°   r¯   rG   c                ó|   • [        UR                  5      S:X  d   eSU R                  UR                  S   5       S3$ )Nr$   úhl.asin(r   r9   rs   ro   s     rE   Ú_print_OpaqueUnaryFn_asinÚ'HalidePrinter._print_OpaqueUnaryFn_asinµ   r¯   rG   c                ó|   • [        UR                  5      S:X  d   eSU R                  UR                  S   5       S3$ )Nr$   úhl.tan(r   r9   rs   ro   s     rE   Ú_print_OpaqueUnaryFn_tanÚ&HalidePrinter._print_OpaqueUnaryFn_tanº   ry   rG   c                ó|   • [        UR                  5      S:X  d   eSU R                  UR                  S   5       S3$ )Nr$   úhl.tanh(r   r9   rs   ro   s     rE   Ú_print_OpaqueUnaryFn_tanhÚ'HalidePrinter._print_OpaqueUnaryFn_tanh¿   r¯   rG   c                ó|   • [        UR                  5      S:X  d   eSU R                  UR                  S   5       S3$ )Nr$   úhl.atan(r   r9   rs   ro   s     rE   Ú_print_OpaqueUnaryFn_atanÚ'HalidePrinter._print_OpaqueUnaryFn_atanÄ   r¯   rG   c                ó   • [        S5      e©NÚlog2©ÚNotImplementedErrorro   s     rE   Ú_print_OpaqueUnaryFn_log2Ú'HalidePrinter._print_OpaqueUnaryFn_log2É   s   € Ü! &Ó)Ð)rG   c                ó  >• UR                   (       a  [        TU ]	  U5      $ UR                  u  p#U R	                  U R                  U5      5      nU R	                  U R                  U5      5      nU R                  SU SU S35      $ )Nr{   z / r9   )Ú
is_integerrL   Ú_print_FloorDivru   rk   rŽ   rg   )rN   rf   ÚxÚdivrP   s       €rE   rÕ   ÚHalidePrinter._print_FloorDivÌ   sp   ø€ Ø�?�?Ü‘7Ñ*¨4Ó0Ð0à—‘‰ˆØ�O‰O˜DŸL™L¨›OÓ,ˆØ�o‰o˜dŸl™l¨3Ó/Ó0ˆØ�‰ ¨1¨#¨S°°°QÐ7Ó8Ð8rG   c                óš   • [        UR                  5      S:X  d   eU R                  SU R                  UR                  S   5       S35      $ )Nr$   ú	hl.round(r   r9   r|   ro   s     rE   Ú_print_RoundÚHalidePrinter._print_RoundÕ   r   rG   c                ó0   • UR                   u  p#SU SU S3$ )NÚ(ú) / (z+hl.f32(0)))ru   )rN   rf   rš   r›   s       rE   Ú_print_IntTrueDivÚHalidePrinter._print_IntTrueDivÜ   s"   € Ø�y‰y‰ˆà�1�#�U˜1˜#˜[Ð)Ð)rG   c                ó€   • UR                   u  p#U R                  U5      n[        U5      nSSU* -  < SU SSU-  < S3$ )Nrn   g      $@z)*hl.round((z	)*hl.f32(ú)))ru   rv   r;   )rN   rf   rC   Úns       rE   Ú_print_RoundDecimalÚ!HalidePrinter._print_RoundDecimalá   sJ   € Ø—‘‰ˆà�k‰k˜#ÓˆÜ�‹FˆØ˜ 1 "™Ñ(¨°S°E¸À4ÈÁ7Á+ÈRÐPÐPrG   rR   )$rV   rW   rX   rY   Ústaticmethodrg   rk   rp   rw   r}   Ú_print_FloorToIntr‚   Ú_print_TruncToIntr†   rŠ   r’   rœ   r¡   r¥   r©   r­   r²   r¶   rº   r¾   rÂ   rÆ   rÊ   rÑ   rÕ   rÛ   Ú_print_RoundToIntrà   rå   rZ   r[   r\   s   @rE   r^   r^   P   sÆ   ø† Øñ:ó ð:ð ñ0ó ð0ò!ò6ò
Ið
 %ÐòIð
 %ÐòHò
@ò+ò
#ò#òGò
6ò
7ò
7ò
6ò
7ò
7ò
6ò
7ò
7ò
*õ9òIð
 %Ðò*÷
Qð QrG   r^   z	hl.Bool()zhl.BFloat(16)zhl.Float(16)zhl.Float(32)zhl.Float(64)z	hl.Int(8)z
hl.Int(16)z
hl.Int(32)z
hl.Int(64)z
hl.UInt(8)zhl.UInt(16)zhl.UInt(32)zhl.UInt(64)c                ó   • [         U    $ ©N)Ú_halide_type©Údtypes    rE   Úhalide_typerð   þ   s   € Ü˜ÑÐrG   c                ó
  • [        U 5      (       a5  U R                  (       a$  U [        R                  :w  a  [        R                  n U [        R
                  [        R                  4;   a  [        R                  n [        U 5      $ rì   )	r   Ú	is_signedr<   r>   Úint32Úfloat16Úbfloat16Úfloat32rð   rî   s    rE   Úhalide_acc_typer÷     sP   € Ü˜×Ñ 5§?§?°uÄÇÁÓ7KÜ—‘ˆØ”—‘¤§¡Ð/Ó/Ü—‘ˆÜ�uÓÐrG   c                  óv  • \ rS rSr\  SG   SHS jj5       r\SIS j5       r\S 5       r\S 5       r	\S 5       r
\S 5       r\S	 5       r\S
 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       r \S 5       r!\S 5       r"\S  5       r#\S! 5       r$\S" 5       r%\S# 5       r&\S$ 5       r'\S% 5       r(\S& 5       r)\S' 5       r*\S( 5       r+\S) 5       r,\S* 5       r-\S+ 5       r.\S, 5       r/\S- 5       r0\S. 5       r1\S/ 5       r2\S0 5       r3\S1 5       r4\S2 5       r5\S3 5       r6\S4 5       r7\S5 5       r8\S6 5       r9\S7 5       r:\S8 5       r;\S9 5       r<\S: 5       r=\S; 5       r>\S< 5       r?\S= 5       r@\S> 5       rA\S? 5       rB\SJS@ j5       rC\SA 5       rD\SB 5       rE\SC 5       rF\SD 5       rG\          SKSE j5       rHSFrIg)LÚHalideOverridesi
  Nc                óZ   • U[         R                  :X  a  SU  S3$ S[        U5       SU  S3$ )NrÞ   z != 0)ra   rb   r9   )r<   Úboolrð   )rÖ   rï   Ú	src_dtypeÚuse_compute_typess       rE   Úto_dtypeÚHalideOverrides.to_dtype  s9   € ð ”E—J‘JÓØ�q�c˜�=Ð Øœ+ eÓ,Ð-¨R°¨s°!Ð4Ð4rG   c                óê   • U[         R                  [         R                  4;   a  S[        U5       SU  S3n S[        U5       SU  S3nU[         R                  [         R                  4;   a  SU S3nU$ )Nra   rb   r9   zhl.reinterpret(rj   )r<   rô   rõ   rð   )rÖ   rï   rü   Úlines       rE   Úto_dtype_bitcastÚ HalideOverrides.to_dtype_bitcast  ss   € àœŸ™¬¯©Ð7Ó7Øœ; yÓ1Ð2°"°Q°C°qÐ9ˆAØ ¤¨UÓ!3Ð 4°B°q°c¸Ð;ˆØ”U—]‘]¤E§N¡NÐ3Ó3Ø+¨D¨6°Ð3ˆDØˆrG   c                ó8   • U R                  [        U5      U5      $ rì   )rþ   rF   )ÚclsÚvaluerï   s      rE   ÚconstantÚHalideOverrides.constant  s   € à�|‰|œO¨EÓ2°EÓ:Ð:rG   c                ó   • SU  S3$ )Nr¤   r9   rR   ©rÖ   s    rE   ÚabsÚHalideOverrides.abs#  ó   € à˜˜˜1ˆ~ÐrG   c                ó\   • [        U S5      (       d  SU  S3$ SU  SU R                   SU  S3$ )NÚnamezhl.exp(r9   z"hl.fast_exp(hl.cast(hl.Float(32), z)) if z!.type().bits() <= 32 else hl.exp(©Úhasattrr  r
  s    rE   ÚexpÚHalideOverrides.exp'  s@   € ä�q˜&×!Ñ!Ø˜Q˜C˜q�>Ð!Ø3°A°3°f¸Q¿V¹V¸HÐDeÐfgÐehÐhiÐjÐjrG   c                ó   • SU  S3$ r‰   rR   r
  s    rE   ÚsqrtÚHalideOverrides.sqrt-  ó   € à˜!˜˜AˆÐrG   c                ó¦   • [        U S5      (       d	  SU  SU S3$ SU R                   SU S3nSU  SU S	U  S
U  SU SU R                   SU  SU S3$ )Nr  r•   rb   r9   ra   ú	.type(), úhl.select((Ú<ú)|hl.is_nan(ú), ú) if z.type().is_float() else hl.min(r  ©rš   r›   s     rE   ÚminimumÚHalideOverrides.minimum1  ó‚   € ô �q˜&×!Ñ!Ø˜Q˜C˜r !  AÐ&Ð&Ø�q—v‘v�h˜i¨ s¨!Ð,ˆØ˜Q˜C˜q   <°¨s°#°a°S¸¸1¸#¸UÀ1Ç6Á6À(ÐJiÐjkÐilÐlnÐopÐnqÐqrÐsÐsrG   c                ó¦   • [        U S5      (       d	  SU  SU S3$ SU R                   SU S3nSU  SU S	U  S
U  SU SU R                   SU  SU S3$ )Nr  rŸ   rb   r9   ra   r  r  Ú>r  r  r  z.type().is_float() else hl.max(r  r  s     rE   ÚmaximumÚHalideOverrides.maximum9  r"  rG   c                ób   • [        US5      (       a  SUR                   SU S3nSU  SU SU S3$ )Nr  ra   r  r9   r�   rb   r  )rš   r›   r�   s      rE   ÚwhereÚHalideOverrides.whereA  sB   € ä�1�f×ÑØ˜1Ÿ6™6˜( )¨A¨3¨aÐ0ˆAØ˜A˜3˜b   2 a S¨Ð*Ð*rG   c                ó   • SU  S3$ )Nr¨   r9   rR   r
  s    rE   ÚcosÚHalideOverrides.cosG  r  rG   c                ó   • SU  S3$ )Nrµ   r9   rR   r
  s    rE   ÚsinÚHalideOverrides.sinK  r  rG   c                ó   • [        S5      e)NÚlgamma©rI   r
  s    rE   r1  ÚHalideOverrides.lgammaO  ó   € ä˜(Ó#Ð#rG   c                ó   • SU  S3$ )Nzhl.erf(r9   rR   r
  s    rE   ÚerfÚHalideOverrides.erfS  r  rG   c                ó   • SU  S3$ )Nr¬   r9   rR   r
  s    rE   ÚcoshÚHalideOverrides.coshW  r  rG   c                ó   • SU  S3$ )Nr¹   r9   rR   r
  s    rE   ÚsinhÚHalideOverrides.sinh[  r  rG   c                ó   • SU  S3$ )Nr±   r9   rR   r
  s    rE   ÚacosÚHalideOverrides.acos_  r  rG   c                ó   • SU  S3$ )Nz	hl.acosh(r9   rR   r
  s    rE   ÚacoshÚHalideOverrides.acoshc  ó   € à˜1˜#˜QÐÐrG   c                ó   • SU  S3$ )Nr½   r9   rR   r
  s    rE   ÚasinÚHalideOverrides.asing  r  rG   c                ó   • SU  S3$ )Nz	hl.asinh(r9   rR   r
  s    rE   ÚasinhÚHalideOverrides.asinhk  rD  rG   c                ó   • SU  SU S3$ )Nz	hl.atan2(rb   r9   rR   ©rÖ   Úys     rE   Úatan2ÚHalideOverrides.atan2o  ó   € à˜1˜#˜R ˜s !Ð$Ð$rG   c                ó   • SU  S3$ )NrÉ   r9   rR   r
  s    rE   ÚatanÚHalideOverrides.atans  r  rG   c                ó   • SU  S3$ )Nz	hl.atanh(r9   rR   r
  s    rE   ÚatanhÚHalideOverrides.atanhw  rD  rG   c                ó   • [        S5      e)NÚcopysignr2  rL  s     rE   rX  ÚHalideOverrides.copysign{  s   € ä˜*Ó%Ð%rG   c                ó   • [        S5      e)NÚerfinvr2  r
  s    rE   r[  ÚHalideOverrides.erfinv  r4  rG   c                ó   • SU  SU S3$ )Nz	hl.hypot(rb   r9   rR   rL  s     rE   ÚhypotÚHalideOverrides.hypotƒ  rP  rG   c                ó   • [        S5      e)NÚ	nextafterr2  rL  s     rE   ra  ÚHalideOverrides.nextafter‡  s   € ä˜+Ó&Ð&rG   c                ó   • U  SU 3$ ©Nz & rR   r  s     rE   Úlogical_andÚHalideOverrides.logical_and‹  ó   € à��C˜�sˆ|ÐrG   c                ó   • U  S3$ )Nz == 0rR   ©rš   s    rE   Úlogical_notÚHalideOverrides.logical_not�  s   € à��Eˆ{ÐrG   c                ó   • U  SU 3$ ©Nz | rR   r  s     rE   Ú
logical_orÚHalideOverrides.logical_or“  rg  rG   c                ó   • SU  SU S3$ )NrÞ   ú ^ r9   rR   r  s     rE   Úlogical_xorÚHalideOverrides.logical_xor—  s   € à�1�#�S˜˜˜1ˆ~ÐrG   c                ó   • U  SU 3$ rd  rR   r  s     rE   Úbitwise_andÚHalideOverrides.bitwise_and›  rg  rG   c                ó   • SU  3$ )NÚ~rR   ri  s    rE   Úbitwise_notÚHalideOverrides.bitwise_notŸ  s   € à�1�#ˆwˆrG   c                ó   • U  SU 3$ rm  rR   r  s     rE   Ú
bitwise_orÚHalideOverrides.bitwise_or£  rg  rG   c                ó   • U  SU 3$ )Nrq  rR   r  s     rE   Úbitwise_xorÚHalideOverrides.bitwise_xor§  rg  rG   c                ó   • U  SU 3$ )Nz << rR   r  s     rE   Úbitwise_left_shiftÚ"HalideOverrides.bitwise_left_shift«  ó   € à��D˜˜ˆ}ÐrG   c                ó   • U  SU 3$ )Nz >> rR   r  s     rE   Úbitwise_right_shiftÚ#HalideOverrides.bitwise_right_shift¯  r„  rG   c                ó   • SU  SU S3$ )Nzhalide_helpers.rand(rb   r9   rR   ©ÚseedÚoffsets     rE   ÚrandÚHalideOverrides.rand³  s   € à% d V¨2¨f¨X°QÐ7Ð7rG   c                ó   • SU  SU S3$ )Nzhalide_helpers.randn(rb   r9   rR   r‰  s     rE   ÚrandnÚHalideOverrides.randn·  s   € à& t f¨B¨v¨h°aÐ8Ð8rG   c           	     ó    • SU  SU SU SU S3	$ )Nzhalide_helpers.randint64(rb   r9   rR   )rŠ  r‹  ÚlowÚhighs       rE   Ú	randint64ÚHalideOverrides.randint64»  s#   € à*¨4¨&°°6°(¸"¸S¸EÀÀDÀ6ÈÐKÐKrG   c                óŠ   • [         R                  " U S5       S[        R                  R                  R                  SU5       3$ )Nr   ú + Úload_seed_offset)ÚopsÚloadr#   rc   ru   Úseed_offset)r  r‹  s     rE   Ú	load_seedÚHalideOverrides.load_seed¿  s7   € ä—(’(˜4 Ó#Ð$ C¬¯©¯©×(AÑ(AÐBTÐV\Ó(]Ð'^Ð_Ð_rG   c                ó   • SU  S3$ )Nz1./hl.sqrt(r9   rR   r
  s    rE   ÚrsqrtÚHalideOverrides.rsqrtÃ  s   € ð ˜Q˜C˜qÐ!Ð!rG   c                ó   • SU  S3$ )NrÁ   r9   rR   r
  s    rE   ÚtanÚHalideOverrides.tanÈ  r  rG   c                ó   • SU  S3$ )NrÅ   r9   rR   r
  s    rE   ÚtanhÚHalideOverrides.tanhÌ  r  rG   c                ó   • SU  S3$ )Nz3(hl.reinterpret(hl.UInt(32), hl.cast(hl.Float(32), z)) >> 31) != 0rR   r
  s    rE   ÚsignbitÚHalideOverrides.signbitÐ  s   € àDÀQÀCÀ~ÐVÐVrG   c                ó   • U  SU  SU SU 3$ )Nz - hl.trunc(Ú/z)*rR   r  s     rE   ÚfmodÚHalideOverrides.fmodÔ  s!   € ð ��L   1 Q C r¨!¨Ð-Ð-rG   c                ó   • SU  SU S3$ )Nzhl.pow(rb   r9   rR   r  s     rE   ÚpowÚHalideOverrides.powÙ  s   € à˜˜˜2˜a˜S Ð"Ð"rG   c                ó   • [        S5      e)NÚldexpr2  )rÖ   rä   s     rE   r²  ÚHalideOverrides.ldexpÝ  s   € ä˜'Ó"Ð"rG   c                ó   • SU  S3$ )Nzhl.log(r9   rR   r
  s    rE   ÚlogÚHalideOverrides.logá  r  rG   c                ó   • [        S5      erÍ   rÏ   r
  s    rE   rÎ   ÚHalideOverrides.log2å  s   € ä! &Ó)Ð)rG   c                ó   • SU  S3$ )Nz hl.is_inf(hl.cast(hl.Float(32), rã   rR   r
  s    rE   ÚisinfÚHalideOverrides.isinfé  ó   € ð 2°!°°BÐ7Ð7rG   c                ó   • SU  S3$ )Nz hl.is_nan(hl.cast(hl.Float(32), rã   rR   r
  s    rE   ÚisnanÚHalideOverrides.isnanî  r¼  rG   c                ó   • SU  S3$ )NrÚ   r9   rR   r
  s    rE   ÚroundÚHalideOverrides.roundó  rD  rG   c                ó   • SU  S3$ )Nr{   r9   rR   r
  s    rE   ÚfloorÚHalideOverrides.floor÷  rD  rG   c                ó   • SU  SU S3$ )NrÞ   rß   z + hl.f32(0))rR   r  s     rE   Úint_truedivÚHalideOverrides.int_truedivû  s   € à�1�#�U˜1˜#˜]Ð+Ð+rG   c                ó.   • SU R                    SU  SU S3$ )Nz"hl.floor(hl.cast(hl.Float(max(32, ú.type().bits())), ú) / r9   ©r  r  s     rE   ÚfloordivÚHalideOverrides.floordivÿ  s)   € ð 1°·±°Ð8JÈ1È#ÈTÐRSÐQTÐTUÐVð	
rG   c                ó<  • [         R                  " [         R                  " SU5      [        R                  5      n[         R                  " [         R                  " US5      [        R                  5      n[         R
                  " X#5      nSUR                   SU S3$ )NÚ0ra   r  r9   )r™  rþ   Últr<   Úint8Úsubr  )r  rÖ   ÚleftÚrightrÓ  s        rE   ÚsignÚHalideOverrides.sign  se   € ä�|Š|œCŸFšF 3¨›N¬E¯J©JÓ7ˆÜ—’œSŸVšV A s›^¬U¯Z©ZÓ8ˆÜ�gŠg�dÓ"ˆØ˜!Ÿ&™&˜ ¨3¨%¨qÐ1Ð1rG   c                ó   • SU  S3$ )Nr�   r9   rR   r
  s    rE   ÚtruncÚHalideOverrides.trunc  rD  rG   c                ó.   • SU R                    SU  SU S3$ )Nz"hl.trunc(hl.cast(hl.Float(max(32, rÊ  rË  r9   rÌ  r  s     rE   ÚtruncdivÚHalideOverrides.truncdiv  s)   € ð
 1°·±°Ð8JÈ1È#ÈTÐRSÐQTÐTUÐVð	
rG   c                ó   • SU  S3$ )Nr…   r9   rR   r
  s    rE   ÚceilÚHalideOverrides.ceil  r  rG   c                ó   • SU  S3$ )NrŸ   z, 0)rR   r
  s    rE   ÚreluÚHalideOverrides.relu  s   € à˜˜˜4Ð Ð rG   c                ó|  • [         R                  R                  U5      n[         R                  R                  [         R                  R	                  U5      [         R                  R                  U5      [        U5      S9nU[        R                  [        R                  4;  a  [        R                  " XB5      $ U$ )N)Úbounds)r#   rc   Úprepare_indexingÚgenfuncÚindex_to_strÚused_dims_from_indexr   r<   ró   r>   r™  rþ   )r  rf   rï   ÚindexÚvars        rE   Ú
index_exprÚHalideOverrides.index_expr!  s†   € ä—‘×)Ñ)¨$Ó/ˆÜ�h‰h×ÑÜ�H‰H×!Ñ! %Ó(Ü�H‰H×)Ñ)¨%Ó0Ü(¨Ó.ð ð 
ˆð
 œŸ™¤e§k¡kÐ2Ó2Ü—<’< Ó+Ð+Øˆ
rG   c                ó®   • [         R                  " U[        R                  5      n[         R                  " XU5      nX!l        [        [        U5      5      $ rì   )r™  rþ   r<   ró   Úhalide_clampÚindirect_indexing_sizer    Ústr)r  Ú	index_varÚsizeÚcheckÚwrap_negs        rE   Úindirect_indexingÚ!HalideOverrides.indirect_indexing-  s?   € ô —L’L ¬E¯K©KÓ8ˆ	Ü×$Ò$ Y°eÓ<ˆ	Ø+/Ô(Ü!¤# i£.Ó1Ð1rG   c                ó  • [         R                  R                  [         R                  R                  U5      S-
  5      n[	        U[
        [        R                  45      (       d  SUR                   SU S3nSU SU S3$ )Nr$   ra   r  r9   z	hl.clamp(z, 0, )	r#   rc   ÚkexprÚrename_indexingr:   r;   r–   ÚIntegerr  )r  r  ró  rô  Úends        rE   rï  ÚHalideOverrides.halide_clamp5  sm   € ä�h‰h�n‰nœQŸX™X×5Ñ5°dÓ;¸aÑ?Ó@ˆÜ˜$¤¤e§m¡mÐ 4×5Ñ5Ø˜UŸZ™Z˜L¨	°#°°aÐ8ˆCð ˜5˜'  s e¨1Ð-Ð-rG   c                ó¬  • [         R                  R                  X5       nU" 5       nS S S 5        WR                  R                  (       a  [        U5      n[         R                  R                  SUR                   S[        U5       S3/ [        R                  " U5      UR                  S9n[        R                  " WXB5      $ ! , (       d  f       N£= f)Nra   r  r9   ©rå  Úshape)r#   rc   Ú
mask_loadsrå  Úis_boolrû   rç  r  rF   r   Úwrapr   r™  r(  )ÚmaskÚbodyÚotherÚnew_maskÚresults        rE   ÚmaskedÚHalideOverrides.masked>  s¥   € ä�X‰X× Ñ  Ô-°Ù“VˆF÷ .ð �=‰=× × Ü˜“KˆEô —‘× Ñ Ø�v—{‘{�m 9¬_¸UÓ-CÐ,DÀAÐFØÜ×#Ò# EÓ*Ø—,‘,ð	 !ð 
ˆô �yŠy˜ 6Ó1Ð1÷ .Õ-ús    CÃ
Cc                ó   • [        S5      e)NÚfrexprÏ   r
  s    rE   r  ÚHalideOverrides.frexpP  s   € ä! 'Ó*Ð*rG   c                ó   • [        S5      e)NÚdevice_assert_asyncrÏ   )ÚcondÚmsgs     rE   r  Ú#HalideOverrides.device_assert_asyncT  s   € ä!Ð"7Ó8Ð8rG   c                ó   • [         erì   rÏ   )r  Úreduction_typer  Ú
extra_metas       rE   Úpartial_accumulateÚ"HalideOverrides.partial_accumulateX  s
   € ô "Ð!rG   rR   ©NT)rï   útorch.dtyperü   úOptional[torch.dtype])rï   r  rü   r  )TT)
r  rñ  r  rñ  r  r&   r  zdict[str, Any]rT   rU   )JrV   rW   rX   rY   rç   rþ   r  Úclassmethodr  r  r  r  r   r%  r(  r+  r.  r1  r6  r9  r<  r?  rB  rF  rI  rN  rR  rU  rX  r[  r^  ra  re  rj  rn  rr  ru  ry  r|  r  r‚  r†  rŒ  r�  r”  rœ  rŸ  r¢  r¥  r¨  r¬  r¯  r²  rµ  rÎ   rº  r¾  rÁ  rÄ  rÇ  rÍ  rÖ  rÙ  rÜ  rß  râ  rì  rö  rï  r	  r  r  r  rZ   rR   rG   rE   rù   rù   
  s�  † Øð ,0Øð	5àð5ð )ô5ó ð5ð óó ðð ñ;ó ð;ð ñó ðð ñkó ðkð
 ñó ðð ñtó ðtð ñtó ðtð ñ+ó ð+ð
 ñó ðð ñó ðð ñ$ó ð$ð ñó ðð ñó ðð ñó ðð ñó ðð ñ ó ð ð ñó ðð ñ ó ð ð ñ%ó ð%ð ñó ðð ñ ó ð ð ñ&ó ð&ð ñ$ó ð$ð ñ%ó ð%ð ñ'ó ð'ð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñ8ó ð8ð ñ9ó ð9ð ñLó ðLð ñ`ó ð`ð ñ"ó ð"ð ñó ðð ñó ðð ñWó ðWð ñ.ó ð.ð ñ#ó ð#ð ñ#ó ð#ð ñó ðð ñ*ó ð*ð ñ8ó ð8ð ñ8ó ð8ð ñ ó ð ð ñ ó ð ð ñ,ó ð,ð ñ
ó ð
ð ñ2ó ð2ð ñ ó ð ð ñ
ó ð
ð ñó ðð ñ!ó ð!ð ñ	ó ð	ð ó2ó ð2ð ñ.ó ð.ð ñ2ó ð2ð" ñ+ó ð+ð ñ9ó ð9ð ð"Øð"àð"ð ð"ð #ð	"ð
 
ó"ó ó"rG   rù   Úhalidec                  ó‚   ^ • \ rS rSr\R
                  " S5      r  S	       S
U 4S jjjrS rS r	SS jr
S rSrU =r$ )ÚHalideCSEVariableif  z\b(tmp\d+)\[\?\]c                ó0   >• [         TU ]  XX4S9  S U l        g ©N©r   )rL   rM   Ú	used_dims)rN   r  rå  rï   r   rP   s        €rE   rM   ÚHalideCSEVariable.__init__i  s   ø€ ô 	‰Ñ˜ uÐÑ:Ø7;ˆ�rG   c                ón  • [        U R                  =(       d    S5      n[        R                  " X#R	                  5       5       HK  n[        U[        5      (       d  M  UR                  c	   XU45       eUR                  UR                  5        MM     [        R                  R                  U5      U l        g )NrR   )r   r"  Ú	itertoolsÚchainÚvaluesr:   r  Úupdater#   rc   Úsort_used_dims)rN   r  ru   ÚkwargsÚusedÚargs         rE   Úupdate_on_argsÚ HalideCSEVariable.update_on_argss  s~   € Ü˜$Ÿ.™.×.¨BÓ/ˆÜ—?’? 4¯©«Ö9ˆCÜ˜#Ô0×1Ó1Ø—}‘}Ñ0ÐC°4¸dÐ2CÓCÐ0Ø—‘˜CŸM™MÖ*ñ :ô Ÿ™×0Ñ0°Ó6ˆ�rG   c                óœ   • [        U5      S:X  a  U R                   S3$ U R                   SSR                  [        [        U5      5       S3$ )Nr   z[()]Ú[rb   Ú])rt   r  ÚjoinÚmaprñ  )rN   Údimss     rE   Ú	index_strÚHalideCSEVariable.index_str{  sE   € Üˆt‹9˜‹>Ø—i‘i�[ Ð%Ð%à—)‘)�˜A˜dŸi™i¬¬C°«Ó7Ð8¸Ð:Ð:rG   c                óp   • U R                   c  U R                   S3$ U R                  U R                   5      $ )Nz[?])r"  r  r5  )rN   s    rE   Ú__str__ÚHalideCSEVariable.__str__�  s0   € Ø�>‰>Ñ!à—i‘i�[ Ð$Ð$Ø�~‰~˜dŸn™nÓ-Ð-rG   c           	     óà   • U R                   b!  [        S U R                    5       5      (       d   eU R                  U R                    Vs/ s H  o!R                  X"5      PM     sn5      $ s  snf )Nc              3  óV   #   • U  H  n[        U[        R                  5      v •  M!     g 7frì   ©r:   r–   ÚExpr©Ú.0rÖ   s     rE   Ú	<genexpr>Ú-HalideCSEVariable.subs_str.<locals>.<genexpr>ˆ  s!   é € ð 2
Ú/=¨!ŒJ�qœ%Ÿ*™*×%Ð%ª~ùó   ‚'))r"  Úallr5  Úget)rN   Úreplacementsrä   s      rE   Úsubs_strÚHalideCSEVariable.subs_str‡  sc   € Ø�~‰~Ñ)¬cñ 2
Ø/3¯~ª~ó2
÷ /
ñ /
ð 	
ð 
ð �~‰~¸t¿~º~ÓNº~¸!×/Ñ/°Ö5¹~ÑNÓOÐOùÒNs   Á
A+)r"  )NN)rå  zValueRanges[Any]rï   r  r   r7   rT   rU   )rT   rñ  )rV   rW   rX   rY   ÚreÚcompileÚundefined_rerM   r-  r5  r8  rF  rZ   r[   r\   s   @rE   r  r  f  sg   ø† Ø—:’:Ð1Ó2€Lð (,Ø $ð<ð !ð<ð %ð	<ð
 ð<ð 
÷<ð <ò7ò;ô.÷Pð PrG   r  c                  óV   ^ • \ rS rSr% S\S'   S\S'   S\S'   S
U 4S jjrSS jrS	rU =r$ )ÚDimensionInfoiŽ  zOptional[sympy.Expr]rf   ú
sympy.Exprró  Ústridec                ó°   >• [         TU ]  5         [        R                  R                  R                  US5      (       a  U* nU* nXl        X l        X0l        g ©Nr   )	rL   rM   r#   ÚgraphÚsizevarsÚstatically_known_ltrf   ró  rN  )rN   rf   ró  rN  rP   s       €rE   rM   ÚDimensionInfo.__init__”  sH   ø€ Ü‰ÑÔÜ�7‰7×Ñ×/Ñ/°¸×:Ñ:Ø�WˆFØ�5ˆDØŒ	ØŒ	Ø�rG   c                ó   • U R                   c   eU R                   nU(       a  US:X  a  gU(       aÀ  0 UEnUR                   H¡  n[        U[        R                  5      (       d  M$  [        U[        R                  5      (       d   e[        R                  R                  UR                  5      n[        U[        5      (       d   e[        UR                  U5      5      X'   M£     [        X15      n[        R                  R!                  U5      $ )Nr   úhl.Var())rf   Úfree_symbolsr   r   ÚTMPr:   r–   ÚSymbolr#   rc   Úlookup_cse_varr  r  r    rF  r!   rè  )rN   rE  Ú	zero_varsrf   Úsymrë  s         rE   r5  ÚDimensionInfo.index_str�  sÏ   € Ø�y‰yÑ$Ð$Ð$Ø�y‰yˆÞ˜ ›ØÞØ+˜lÐ+ˆLà×(Ô(�Ü! #¤t§x¡x×0Ó0Ü% c¬5¯<©<×8Ñ8Ð8Ð8ÜŸ(™(×1Ñ1°#·(±(Ó;�CÜ% cÔ+<×=Ñ=Ð=Ð=Ü(:¸3¿<¹<ÈÓ;UÓ(V�LÓ%ñ )ô ˜dÓ1ˆDÜ�x‰x×$Ñ$ TÓ*Ð*rG   )rf   ró  rN  rS   ©NF)	rV   rW   rX   rY   Ú__annotations__rM   r5  rZ   r[   r\   s   @rE   rL  rL  Ž  s$   ø‡ à
ÓØ
ÓØÓ÷÷+ò +rG   rL  c                óŠ  • [         R                  R                  R                  X5      (       a  g [         R                  R                  R	                  U 5      n[         R                  R                  R	                  U5      nX#:X  a)  [         R                  R                  R                  X5        X#:H  $ ! [
         a     gf = f©NTF)r#   rQ  rR  Ústatically_known_equalsÚsize_hint_or_throwÚ	TypeErrorÚcheck_equals)rÔ  rÕ  rš   r›   s       rE   Úeqrf  ¯  s“   € Ü‡w�w×Ñ×/Ñ/°×<Ñ<ØðÜ�G‰G×Ñ×/Ñ/°Ó5ˆÜ�G‰G×Ñ×/Ñ/°Ó6ˆð 	ƒvÜ	�‰×Ñ×%Ñ% dÔ2Ø‰6€Møô	 ó Ùðús   ±AB5 Â5
CÃCc                óÌ  • [         R                  R                  R                  X5      (       a  g [         R                  R                  R	                  U 5      n[         R                  R                  R	                  U5      nX#:  a)  [         R                  R                  R                  X5        X#:  $ ! [
         a$    [        R                  " X5      nX@:X  a  X:g  s $  gf = fra  )	r#   rQ  rR  rS  rc  rd  r–   ÚgcdÚcheck_lt)rÔ  rÕ  rš   r›   rh  s        rE   rÑ  rÑ  ¼  s®   € Ü‡w�w×Ñ×+Ñ+¨D×8Ñ8ØðÜ�G‰G×Ñ×/Ñ/°Ó5ˆÜ�G‰G×Ñ×/Ñ/°Ó6ˆð 	ƒuÜ	�‰×Ñ×!Ñ! $Ô.Ø‰5€Løô ó Ü�iŠi˜Ó$ˆØ‹;Ø‘=Ò Ùð	ús   ±AB5 Â5)C#Ã"C#c                  óâ  ^ • \ rS rSr% \r\rS\S'       S&U 4S jjr	S'S jr
S(S jrS)U 4S jjrS	 rS
 r  S*U 4S jjrS rS+S jrS rS rS*S jrS rS,S jrS-S jrS.S jr S/         S0S jjr          S1S jrS r        S2S jr\R<                  " 5       SS.   S3S jjrSS.S3S jjr S.S jr!S r"S4S jr#S/S  jr$\%S! 5       r&S5S6S" jjr'S# r(        S7S$ jr)S%r*U =r+$ )8ÚHalideKerneliÌ  zCallable[[sympy.Expr], str]rù  c                óz  >• [         TU ]  " U40 UD6  U R                  U l        U R                  U l        U R                  U l        [        5       U l        U R                  U l	        U R                  U l
        0 U l        0 U l        0 U l        0 U l        0 U l        0 U l        [#        [$        5      U l        SU l        g r^  )rL   rM   r  ÚcomputeÚloadsÚstoresr(   Úindexing_code_domÚinside_reductionÚneeds_dom_indexingÚhas_reductionÚbuffer_dimensionsÚbuffer_offsetsÚhalide_varsÚindex_replacementsÚreduction_renamesÚdom_renamesr   ÚlistÚbuffer_aliasesÚhas_indirect_indexing)rN   Útilingr*  rP   s      €rE   rM   ÚHalideKernel.__init__Ð  s¤   ø€ ô
 	‰Ò˜Ñ* 6Ò*à—y‘yˆŒØ—Y‘YˆŒ
Ø—i‘iˆŒÜ!/Ó!1ˆÔØ"&×"7Ñ"7ˆÔØ!×2Ñ2ˆÔØACˆÔØ57ˆÔà;=ˆÔà@BˆÔàCEˆÔàHJˆÔä4?ÄÓ4EˆÔØ%*ˆÕ"rG   c                ó   • [        U5      $ rì   )rð   )rN   rï   s     rE   Údtype_to_strÚHalideKernel.dtype_to_strë  s   € Ü˜5Ó!Ð!rG   Nc                ó^   • U R                   R                  U SU< S35        [        XX45      $ )Nz = hl.Func(r9   )r  Ú	writeliner  )rN   r  rå  rï   r   s        rE   Úcreate_cse_varÚHalideKernel.create_cse_varï  s.   € Ø�	‰	×Ñ˜t˜f K°©x°qÐ9Ô:ä  ¨uÓ<Ð<rG   c           
     óJ  >^^^• U R                   (       d"  U R                  (       d  U R                  (       a   e[        R                  " [
        R                  R                  R                  [        S9m[        R                  [        [        T U ]8  U5      5      n[        [            " 5       m["        R$                  R'                  U R(                   Vs/ s H  o"R*                  R-                  5       PM     sn5       Vs0 s H  nUR/                  5       U_M     snmS nUU4S jnUU4S jnU Hü  nUR1                  [2        5      (       aY  UR5                  [3        [6        R8                  " S5      [6        R8                  " S5      [6        R8                  " S5      5      U5        UR1                  [:        5      (       aD  UR5                  [;        [6        R8                  " S5      [6        R8                  " S5      5      U5        TR=                  [        T U ]9  U5      R>                  5        Mþ     [A        S T 5       5      U l!        S	n[E        U R(                  5       GH¡  nUR*                  R-                  5        Vs/ s H  o3R/                  5       T;   d  M  UPM     n	nU	RG                  U4S
 jS9  U	(       d+  U	RI                  URK                  SURL                  5      5        Sn
[6        RN                  RP                  n/ nU
[S        U	5      :  Ga  [U        URL                  U5      (       Gdì  U	 Vs/ s H2  n[U        URV                  U5      (       d  M   U" URX                  5      PM4     nnU
[S        U5      -  n
U(       d   U	5       eU[        RZ                  " [
        R                  R                  R\                  U5      -  nUR_                  U	 Vs/ s HQ  n[a        X³RV                  5      (       d  M  [a        URV                  U5      (       d  M<  U" URV                  U-  5      PMS     sn5        U(       Ga²  [        RZ                  " [6        Rb                  U5      n[U        US5      (       a7  U" URL                  U-  5      n[U        US5      (       a   e/ n[S        U	5      n
Sn[e        S[S        U R                  5       35      nURf                  (       a.  [e        S[S        U R                  5       35      U R                  U'   XðR                  U'   URI                  UU45        X¿-  nU	 Vs/ s H,  n[U        URV                  U5      (       d  M   URX                  PM.     nnU
[S        U5      -  n
[S        U5      nU Vs/ s H/  n[U        UU5      (       a  M  [6        Rh                  " UU-  5      PM1     nn[S        U5      U:  d  US:X  d   eUR_                  U5        U(       a  GM²  U
[S        U	5      :  a  [U        URL                  U5      (       d  GMì  U	 HÝ  n SnSn[U        URV                  U5      (       d/  UU   u  nnUS-  nUU-  n[U        URV                  U5      (       d  M/  Sn[6        RN                  Rj                  n[U        URX                  U5      (       d7  UU   u  nnUS-  nUUU-  -  nUU-  n[U        URX                  U5      (       d  M7  UU R                   UR/                  5       '   Mß     GM¤     U R                   H/  nU Rp                  Rs                  U SURt                  < S35        M1     U R                  (       aO  U Rw                  SU R                  Ry                  5        VVs0 s H  u  nnUU R                  U   _M     snn5        ggs  snf s  snf s  snf s  snf s  snf s  snf s  snf ! [l         aÉ    U(       d   e[6        RN                  Rj                  n[6        RN                  RP                  nU H  u  nnUUU-  -  nUU-  nM     [
        R                  R                  Ro                  [3        UURV                  URX                  5      U R                  5      U R                   UR/                  5       '    GM{  f = fs  snnf )a¹  
Hook called right before codegen with every index that will be
used in the fused kernel.

This populates self.halide_vars/index_replacements/reduction_renames which is an alternate indexing
scheme that avoids using divide and modulus.  Instead of xindex/yindex/rindex
we base indexing on a larger number of vars whose product combines to those.

This function populates self.halide_vars, self.index_replacements, and self.reduction_renames
©Úfallbackc                ó|   • [         R                  " [        R                  R                  R                  U 5      5      $ rì   )r–   Úsimplifyr#   rQ  rR  Úremove_precomputed_replacementsre   s    rE   rŠ  Ú0HalideKernel.finalize_indexing.<locals>.simplify  s+   € Ü—>’>Ü—‘× Ñ ×@Ñ@ÀÓFóð rG   c                ó"  >• U T;   aˆ  TU    nTR                  UR                  R                  UR                  U-  [        R
                  R                  R                  U[        UR                  U5      5      5      R                  5       5        g g rì   )ÚaddÚrootÚlookupÚdivisorr#   rQ  rR  Úevaluate_minr   ÚlengthÚsymbol)Úbaser‘  ÚmodulusÚnodeÚall_used_symbolsÚsym_to_nodes       €€rE   Úvisit_modular_indexingÚ>HalideKernel.finalize_indexing.<locals>.visit_modular_indexing  sw   ø€ Ø�{Ó"Ø" 4Ñ(�Ø ×$Ñ$Ø—I‘I×$Ñ$ØŸ™ wÑ.ÜŸ™×(Ñ(×5Ñ5Ø#¤X¨d¯k©k¸7Ó%Cóó÷
 ‘f“hõð #rG   c           	     óÒ   >• U T;   a`  TU    nTR                  UR                  R                  UR                  U-  [	        UR
                  U5      5      R                  5       5        g g rì   )rŽ  r�  r�  r‘  r   r“  r”  )r•  r‘  r—  r˜  r™  s      €€rE   Úvisit_floor_divÚ7HalideKernel.finalize_indexing.<locals>.visit_floor_div  s]   ø€ Ø�{Ó"Ø" 4Ñ(�Ø ×$Ñ$Ø—I‘I×$Ñ$ØŸ™ wÑ.Ü  §¡¨gÓ6ó÷ ‘f“hõ	ð #rG   r•  r‘  r–  c              3  óV   #   • U  H  n[        U[        R                  5      v •  M!     g 7frì   )r   r   ÚINDIRECT)r?  r\  s     rE   r@  Ú1HalideKernel.finalize_indexing.<locals>.<genexpr>=  s"   é € ð )
Ú:J°3ŒN˜3¤§¡×.Ð.Ò:JùrB  Fc                ó(   >• T" U R                   5      $ rì   )r‘  )rä   Ú	size_hints    €rE   Ú<lambda>Ú0HalideKernel.finalize_indexing.<locals>.<lambda>D  s   ø€ ¡Y¨q¯y©yÔ%9rG   ©Úkeyr$   r   TÚhÚhrz
 = hl.Var(r9   ÚrdomN)=rw  rv  rx  Ú	functoolsÚpartialr#   rQ  rR  r£  r   ÚdictÚfromkeysr3  rL   ræ  r   r   r%  r&  Úfrom_iterableÚrange_treesÚnodesr'  r”  Úhasr   Úreplacer–   ÚWildr   r(  rW  Úanyr|  ÚreversedÚsortÚappendr�  ÚnumelÚSÚOnert   rf  r‘  r“  ÚreduceÚevaluate_maxÚextendrÑ  rh  r    Úis_reductionrŠ  ÚZeroÚ
IndexErrorÚsimplify_with_rangesÚindexing_coderƒ  r  Úcodegen_rdomÚitems)!rN   ÚindicesÚtreerä   rŠ  rš  r�  rê  Úhad_fallbackr±  Úhandled_countr‘  Úadded_sym_sizeÚsizes_to_addrü  Ú	next_sizer\  Ú	new_sizesÚ	prior_lenÚsr—  Úidxró  r“  rf   Ú
full_indexrN  ÚvÚrvr˜  r£  r™  rP   s!                                @@@€rE   Úfinalize_indexingÚHalideKernel.finalize_indexingô  s&  û€ ð ×#×# t×'7×'7¸4×;Q×;Qð	
ð 
ô ×%Ò%¤a§g¡g×&6Ñ&6×&@Ñ&@Ì3ÑOˆ	ä—-‘-¤¤E¡GÑ$<¸gÓ FÓGˆÜ%¤cš?Ó,Ðô —_‘_×2Ñ2Ø15×1AÒ1AÓBÒ1A¨—‘×"Ñ"Ö$Ñ1AÑBôó
ò�ð �H‰H‹J˜ŠMññ
ˆò	ö

	ö	ó ˆEØ�y‰yœ×)Ñ)Ø—‘Ü#ÜŸ
š
 6Ó*ÜŸ
š
 9Ó-ÜŸ
š
 9Ó-óð
 +ôð �y‰yœ×"Ñ"Ø—‘ÜÜŸ
š
 6Ó*ÜŸ
š
 9Ó-óð $ôð ×#Ñ#¤E¡GÑ$<¸UÓ$C×$PÑ$PÖQñ% ô( &)ñ )
Ù:Jó)
ó &
ˆÔ"ð ˆÜ˜T×-Ñ-×.ˆDØ $§
¡
× 1Ñ 1Ô 3ÓVÒ 3˜1·x±x³zÐEUÑ7U—QÑ 3ˆEÐVØ�J‰JÔ9ˆJÑ:ÞØ—‘˜TŸ[™[¨¨D¯J©JÓ7Ô8ØˆMÜ—g‘g—k‘kˆGØˆNà¤# e£*Ô,´R¸¿
¹
ÀG×5LÒ5Lá05ó Ú05¨1¼¸A¿I¹IÀw×9OÓ&‘H˜QŸX™XÖ&±ð ð  ð ¤ \Ó!2Ñ2�Þ#Ð* UÓ*�|Ø¤	× 0Ò 0Ü—G‘G×$Ñ$×1Ñ1°<ó!ñ �ð ×#Ñ#ñ "'óâ!&˜AÜ˜g§y¡y×1ó 6ä68¸¿¹ÀC×6Hó 6™ §¡¨WÑ!4Ö5Ù!&ñô÷ #Ü )× 0Ò 0´·±¸LÓ I�IÜ˜) Q×'Ñ'ñ %-¨T¯Z©Z¸'Ñ-AÓ$B˜	Ü#% i°×#3Ñ#3Ð3Ð3Ø')˜Ü(+¨E«
˜Ø'+˜Ü,¨q´°T×5EÑ5EÓ1FÐ0GÐ-HÓI�Cà×(×(Ü6HØ ¤ T×%5Ñ%5Ó!6Ð 7Ð8ó7˜×.Ñ.¨sÑ3ð -6×$Ñ$ SÑ)Ø"×)Ñ)¨3°	Ð*:Ô;ØÑ(�GÙ38Ó S²5¨a¼B¸q¿y¹yÈ'×<R£ §¤±5�IÐ SØ!¤S¨£^Ñ3�MÜ # LÓ 1�Iñ ".ó$â!-˜AÜ! ! Y×/ó 6œŸš q¨9¡}Ö5Ù!-ð !ð $ô
 ˜|Ó,¨yÓ8¸IÈ»NÐJÐJØ ×'Ñ'¨	Ô2÷9 #‘lð#  ¤# e£*Ó,´R¸¿
¹
ÀG×5LÔ5Ló` �ðØ�CØ�GÜ  §¡¨w×7Ñ7Ø$2°3Ñ$7™	˜˜TØ˜q™˜Ø 4™˜ô ! §¡¨w×7Ó7ð �FÜ Ÿ7™7Ÿ<™<�DÜ  §¡¨f×5Ñ5Ø$2°3Ñ$7™	˜˜TØ˜q™˜Ø ¨¡Ñ,˜Ø $™˜ô	 ! §¡¨f×5Ó5ð
 >B�D×+Ñ+¨D¯K©K«MÓ:ô ñs /ðp ×#Ô#ˆCØ×Ñ×(Ñ(¨C¨5°
¸3¿8¹8¹,ÀaÐ)HÖIñ $à×!×!Ø×ÑØØ6:×6LÑ6L×6RÑ6RÔ6TÔUÒ6T©U¨Q°��T×%Ñ% aÑ(Ò(Ñ6TÒUõð "ùòg Cùò
ùòz Wùò ùòùò4 !Tùò$øô2 "ó Þ'Ð'˜<Ü!&§¡§¡�JÜ"ŸW™WŸ[™[�FÛ%3™	˜˜TØ" f¨s¡lÑ2˜
Ø $™šñ &4ô Ÿ™×(Ñ(×=Ñ=Ü+¨J¸¿¹ÀdÇkÁkÓRØ ×,Ñ,óð ×+Ñ+¨D¯K©K«MÕ:ðüó( Vsu   Ã#^&Ä^+Ê ^0Ê^0Ì6^5Í^5Ï^:
Ï:^:
Ð^:
Ô5^?Õ^?Ö_Ö _Ø$A_	Ù2A,_	Û _	Ý?bß	Cbâbc           
     ó"  • U R                   (       a  SOSnXR                  ;   a  U R                  U   $ 0 nU R                   Hp  nU R                   (       d  X0R                  ;   a  M%  [        R
                  " SUR                  5      nU(       d   e[        SU UR                  S5       35      X#'   Mr     U R                  U S3UR                  5        VVs0 s H  u  pVX`R                  U   _M     snn5        X R                  U'   U$ s  snnf )zCRDom based indexing uses explicit iteration ranges for Func updatesÚiÚoz^h(\d+)$r¨  r$   Údom)rq  ry  rv  rx  rH  Úmatchr  r    ÚgrouprÄ  rÅ  )rN   ÚprefixÚrenamesrë  ÚmrÒ  rÓ  s          rE   Úsetup_dom_indexingÚHalideKernel.setup_dom_indexing¢  sï   € à×-×-‘°3ˆØ×%Ñ%Ó%Ø×#Ñ# FÑ+Ð+àˆØ×#Ô#ˆCØ×(×(¨S×4JÑ4JÓ-JÙÜ—’˜ c§h¡hÓ/ˆAÞˆH�1Ü-°°&°¸!¿'¹'À!»*¸Ð.FÓGˆG‹Lñ $ð 	×ÑØˆh�cˆNÀ'Ç-Á-Ä/ÔRÂ/¹¸˜R×!1Ñ!1°!Ñ!4Ò4Á/ÒRô	
ð $+×Ñ˜Ñ Øˆùó	 Ss   ÃDc           	     ó~  • UR                  5        Vs/ s H'  nSU R                  U R                  U5      5       S3PM)     nnU R                  R	                  U SSR                  U5       S35        [        UR                  5       5       H)  u  pVU R                  R	                  U SU SU S35        M+     g s  snf )	Núhl.Range(0, r9   z = hl.RDom([rb   ú])ú = r0  r1  )r'  rù  rú  rÃ  rƒ  r2  Ú	enumerateÚkeys)rN   r  Úvarsró  Úrsizesr×  Úrsyms          rE   rÄ  ÚHalideKernel.codegen_rdom·  s¸   € ð Ÿ™œó
â%�ð ˜4Ÿ:™: d×&:Ñ&:¸4Ó&@ÓAÐBÀ!ÓDÙ%ð 	ð 
ð 	×Ñ×$Ñ$¨ v¨\¸$¿)¹)ÀFÓ:KÐ9LÈBÐ%OÔPÜ  §¡£Ö-‰GˆAØ×Ñ×(Ñ(¨D¨6°°T°F¸!¸A¸3¸aÐ)@ÖAò .ùò
s   “.B:c                ó²   >• [         TU ]  U5      n[        XR                  5      n[        R
                  R                  R                  XR                  5      $ rì   )	rL   ræ  r!   rw  r#   rQ  rR  rÂ  rv  )rN   rê  rP   s     €rE   ræ  ÚHalideKernel.prepare_indexingÀ  sE   ø€ ô ‘Ñ(¨Ó/ˆÜ˜5×"9Ñ"9Ó:ˆÜ�w‰w×Ñ×4Ñ4°U×<LÑ<LÓMÐMrG   c                ó¨   • [        U[        R                  5      (       a%  U R                  UR                  5      R
                  $ U R                  U   $ )zThe size of an index symbol)r   r   rX  rZ  r  rð  rv  )rN   r\  s     rE   Úsym_sizeÚHalideKernel.sym_sizeÈ  s?   € ä˜#œtŸx™x×(Ñ(Ø×&Ñ& s§x¡xÓ0×GÑGÐGØ×Ñ Ñ$Ð$rG   c           	     óÞ	  ^ ^^• / m[        UR                  S S9 HŠ  n[        U[        R                  [        R
                  45      (       a  TR                  U5        ME  [        U[        R                  [        R                  [        R                  45      (       a  M…   U5       e   [        R                  R                  n[        R                  T[        R                  R                  5      n/ n[        R                  " T R!                  U5      5      n[#        U[        R$                  5      (       a  UR&                  OU/ Hé  nUR                   V	s/ s H  o™U;   d  M
  U	PM     n
n	[)        U
5      S:X  a  XX-  nM;  [)        U
5      S:X  a  XjS   ==   U-  ss'   M[  / n[+        [)        U5      5       Hm  nX|   c   eX|   u  pÞ[-        U5      [-        U
5      -  (       a/  U
R/                  U V	s/ s H  o™U
;  d  M
  U	PM     sn	5        XŽ-  nM[  UR                  XÞ45        Mo     / UQX¨4PnMë     UU U4S jn/ nU H;  u  nnU H  n	UUR1                  U	5      -  nM     UR                  U" UU5      5        M=     UR3                  5        H  u  nnUR                  U" UU/5      5        M!     UR5                  S S9  U(       dF  T R6                  (       a4  UR                  [9        [        R                  R                  SS5      5        O‡[:        R<                  R>                  RA                  US   RB                  S5      (       dK  URE                  S[9        [        R                  R                  T(       a  SOUS   RB                  S5      5        U(       aÈ  T(       dÁ  UT RF                  ;   an  [:        R<                  R>                  RI                  UT RF                  U   5      (       a2  T RK                  UUT RF                  U   -
  5        T RF                  U   nOC[:        R<                  R>                  RM                  US5      (       a  T RK                  UU5        SnUn[N        RP                  " 5        He  nT RS                  UUUT5      (       a  UU4s  $ T(       a   eU SU 3nUT RT                  U   ;  d  MG  T RT                  U   R                  U5        Mg     gs  sn	f s  sn	f )	zEConvert address-based indexing into dimensions using self.halide_varsc                ó   • U R                   $ rì   rÌ  r
  s    rE   r¤  Ú5HalideKernel.indexing_to_dimensions.<locals>.<lambda>Ñ  s   € ¸A¿FºFrG   r¦  r   r$   Nc                ó,  >• [         R                  " U 5      n [        U5      S:X  aV  [         R                  " ST
S9nU R	                  X!S   -  5      nU(       a$  [        US   T	R                  US   5      X2   5      $ T(       a   U 5       e[         R                  " [        X Vs0 s H  oDT	R                  U5      S-
  _M     sn5      S-   5      n[         R                  R                  n[        U [         R                  5      (       a|  U R                   Hl  n[        U[         R                  5      (       d  M$  Xg-  n[         R                  " X-  5      n [         R                  " [         R                  " XW-  5      5      nMn     [        XU5      $ s  snf )Nr$   Úwild)Úexcluder   )r–   Úfactorrt   r´  rÚ  rL  rî  rŠ  r!   rº  r»  r:   ÚMulru   rû  Úceiling)rf   ÚsymsÚstride_wildrÞ  r\  r“  rN  ÚtermÚis_storerN   Úsymbolss           €€€rE   Úexpr_to_dimensionÚ>HalideKernel.indexing_to_dimensions.<locals>.expr_to_dimensionõ  s<  ø€ Ü—<’< Ó%ˆDÜ�4‹y˜A‹~Ü#Ÿjšj¨¸ÑA�Ø—J‘J˜{°!©WÑ4Ó5�ÞÜ(Ø˜Q™ §¡¨t°A©wÓ!7¸¹óð ö  Ð% Ó%�<Ü—^’^Ü˜4ÈÓ!NÊÀ# t§}¡}°SÓ'9¸AÑ'=Ò"=ÉÑ!NÓOÐRSÑSóˆFô —W‘W—[‘[ˆFÜ˜$¤§	¡	×*Ñ*Ø ŸIœI�DÜ! $¬¯©×6Ó6Ø™˜Ü$Ÿ~š~¨d©kÓ:˜Ü!&§¢´·²¸f¹mÓ0LÓ!Mšñ	 &ô
 ! ¨vÓ6Ð6ùò "Os   Â$Fc                ón   • [         R                  R                  R                  U R                  [
        S9$ )Nr‡  )r#   rQ  rR  r£  rN  r   )Úds    rE   r¤  rò    s"   € ¤§¡× 0Ñ 0× :Ñ :¸1¿8¹8ÌcÐ :Ñ RrG   Ú_view)+ÚsortedrW  r   r   ÚHALIDErX  r¸  ÚUNBACKED_INTÚSIZEÚPRECOMPUTED_SIZEr–   rº  rÀ  r­  r®  Úexpandrú  r:   ÚAddru   rt   Úranger   r¾  ÚpoprÅ  r·  r|  rL  r#   rQ  rR  rb  rN  Úinsertru  Ústatically_known_geqÚapply_offset_to_dimensionÚstatically_known_gtr%  ÚcountÚinstall_dimsr{  )rN   rë  rê  rü  r\  r‹  Ú
split_exprÚsplit_failedÚpartrÒ  Ú	part_varsÚnew_split_failedr×  Ú
other_varsÚ
other_partrþ  r4  rù  rf   Úorig_varrý  s   `  `                @rE   Úindexing_to_dimensionsÚ#HalideKernel.indexing_to_dimensionsÎ  s  ú€ àˆÜ˜%×,Ñ,Ñ2BÔCˆCÜ˜c¤D§K¡K´·±Ð#:×;Ñ;Ø—‘˜sÖ#ä%Øä×)Ñ)ÜŸ	™	Ü×-Ñ-ð÷ó ð ð óð ñ	 Dô —‘—‘ˆÜ—]‘] 7¬E¯G©G¯L©LÓ9ˆ
ØDFˆÜ—’˜T×1Ñ1°%Ó8Ó9ˆÜ",¨U´E·I±I×">Ñ">�E—J’JÀUÀGÒKˆDØ$(×$5Ò$5ÓIÒ$5˜q¸j¹ŸÑ$5ˆIÐIÜ�9‹~ Ó"Ø‘’Ü�Y“ 1Ó$Ø Q™<Ó(¨DÑ0Õ(à#%Ð Üœs <Ó0Ö1�AØ'™?Ñ6Ð6Ð6Ø-9©_Ñ*�JÜ! *Ó-´
¸9Ó0E×EØ!×(Ñ(±ZÓ)V²Z°ÈIÑCU¯!±ZÑ)VÔWØÑ*šà(×/Ñ/°Ð0HÖIñ 2ð  FÐ!1ÐE°IÐ3DÐE’ñ! L÷$	7ð. ˆÛ&‰JˆD�$Û�Ø˜
Ÿ™ qÓ)Ñ)’ñ à�K‰KÑ)¨$°Ó5Ö6ñ 'ð $×)Ñ)Ö+‰IˆC�Ø�K‰KÑ)¨$°°Ó6Ö7ñ ,à�	‰	ÑRˆ	ÑSæØ×)×)à—‘œM¬%¯'©'¯,©,¸¸1Ó=Ô>øÜ—‘×!Ñ!×9Ñ9¸$¸q¹'¿.¹.È!×LÑLà�K‰KØ”=¤§¡§¡¶H©qÀ$ÀqÁ'Ç.Á.ÐRSÓTôö žØ�d×)Ñ)Ó)¬a¯g©g×.>Ñ.>×.SÑ.SØ˜×+Ñ+¨CÑ0÷/ñ /ð ×.Ñ.¨t°V¸d×>QÑ>QÐRUÑ>VÑ5VÔWØ×,Ñ,¨SÑ1‘Ü—‘×!Ñ!×5Ñ5Ø˜÷ñ ð ×.Ñ.¨t°VÔ<Ø�àˆÜ—’Ö"ˆAØ× Ñ   d¨F°H×=Ñ=Ø˜D�yÒ ÞÐ�<Ø�J˜e A 3Ð'ˆCØ˜$×-Ñ-¨hÑ7Õ7Ø×#Ñ# HÑ-×4Ñ4°SÖ9ò #ùòU Jùò *Ws   Å	S%Å$S%Ç2	S*Ç?S*c                ó|  • XR                   ;  a  X R                   U'   X0R                  U'   gU R                  U   U:w  d%  [        U R                   U   5      [        U5      :w  a  gU(       a  U R                   U   U:H  $ [        U R                   U   U5       H¡  u  pVUR                  UR                  :w  a    gUR
                  UR
                  :w  d  UR                  UR                  :w  d  MW  [        R                  R                  R                  UR
                  UR
                  5      Ul        SUl        M£     g)z>Try to set self.buffer_dimensions[var], return True on successTFN)rt  ru  rt   ÚziprN  ró  rf   r#   rQ  rR  r½  )rN   rë  r4  r‹  rü  ÚoldÚnews          rE   r  ÚHalideKernel.install_dims6  s  € à×,Ñ,Ó,Ø*.×"Ñ" 3Ñ'Ø'-×Ñ Ñ$ØØ×Ñ˜sÑ# vÓ-´Ø×"Ñ" 3Ñ'ó2
ä�‹Yó2ð ÞØ×)Ñ)¨#Ñ.°$Ñ6Ð6Ü˜D×2Ñ2°3Ñ7¸Ö>‰HˆCØ�z‰z˜SŸZ™ZÓ'ÙØ�x‰x˜3Ÿ8™8Ó# s§x¡x°3·8±8Õ';ÜŸ7™7×+Ñ+×8Ñ8¸¿¹À3Ç8Á8ÓL�”Ø�–ñ ?ð rG   c                óŒ  • US:X  a  g [        [        [        U5      5      5       H”  nX   R                  S:X  d=  [        R
                  R                  R                  X!U   R                  5      (       d  MR  [        X!U   R                  5      nX$X   R                  -  -  nX   =R                  U-  sl	        M–     US:X  d   eg )Nr   r$   )
r¶  r
  rt   rN  r#   rQ  rR  r  r   rf   )rN   r4  r‹  r×  r  s        rE   r  Ú&HalideKernel.apply_offset_to_dimensionJ  sž   € Ø�Q‹;ØÜœ%¤ D£	Ó*Ö+ˆAØ‰w�~‰~ Ó"¤a§g¡g×&6Ñ&6×&KÑ&KØ˜Q™Ÿ™÷'ó 'ô   ¨Q©¯©Ó7�Ø ¡§¡Ñ/Ñ/�Ø‘—’ Ñ$—ñ ,ð ˜‹{Ð‰{rG   c                óØ  • [         [        R                     " 5       nUR                   GH-  n[	        U[        R                  5      (       d   e[        U[        R                  5      (       a\  U R                  UR                  5      n[	        U[        5      (       a  UR                  c   eUR                  UR                  5        M   [        U[        R                  5      (       a  UR                  U5        MÒ  [        U[        R                  [        R                   [        R"                  [        R$                  45      (       a  GM"  ['        SU 35      e   U R)                  U5      $ )zIDetect which range trees are used to populate HalideCSEVariable.used_dimszunhandled symbol )r   r–   rY  rW  r:   r   r   rX  rZ  r  r  r"  r(  r  rŽ  r  r  r  ÚINDEXrÐ   r)  )rN   rê  r"  r\  Úcse_vars        rE   ré  Ú!HalideKernel.used_dims_from_indexV  s  € äœuŸ|™|Ò,Ó.ˆ	Ø×%Õ%ˆCÜ˜c¤5§<¡<×0Ñ0Ð0Ð0Ü˜c¤4§8¡8×,Ñ,à×-Ñ-¨c¯h©hÓ7�ä˜wÔ(9×:Ñ:Ø×)Ñ)Ñ5ðð6ð × Ñ  ×!2Ñ!2Ö3Ü ¤T§[¡[×1Ñ1Ø—‘˜cÖ"ÜØ”d×'Ñ'¬¯©´D×4IÑ4IÌ4Ï:É:ÐV÷ñ ò ä)Ð,=¸c¸UÐ*CÓDÐDñ# &ð$ ×"Ñ" 9Ó-Ð-rG   c                ó  • [        S U 5       5      (       d   e[        R                  " U R                  U R                  R                  5       5       Vs/ s H  nX!;   d  M
  UPM     nn[        U5      [        U5      :X  d   eU$ s  snf )Nc              3  óV   #   • U  H  n[        U[        R                  5      v •  M!     g 7frì   r<  r>  s     rE   r@  Ú.HalideKernel.sort_used_dims.<locals>.<genexpr>n  s   é € Ð@²i°”:˜a¤§¡×,Ð,²iùrB  )rC  r%  r&  rv  rx  r'  rt   )rN   r"  r\  Úordereds       rE   r)  ÚHalideKernel.sort_used_dimsm  sŒ   € ÜÑ@±iÓ@×@Ñ@Ð@Ð@ô !—’Ø× Ñ  $×"8Ñ"8×"?Ñ"?Ó"Aôó
ò�ð Ñ÷	 ñð 	ð 
ô �7‹|œs 9›~Ó-Ð-Ð-Øˆùò
s   Á	BÁ"Bc                ó�   ^^• SR                  UU4S jU 5       5      n[        U5      S:X  a  SnU$ [        U5      S:X  a  U S3nU$ )Nrb   c              3  óF   >#   • U  H  oR                  TT5      v •  M     g 7frì   )r5  )r?  r  rE  r[  s     €€rE   r@  Ú.HalideKernel.make_index_str.<locals>.<genexpr>z  s   øé € ÐQÊDÀqŸk™k¨,¸	×BÐBÊDùs   ƒ!r   ú()r$   Ú,)r2  rt   )rN   r4  rE  r[  r5  s     `` rE   Úmake_index_strÚHalideKernel.make_index_stry  sM   ù€ Ø—I‘IÕQÉDÓQÓQˆ	Üˆt‹9˜‹>ØˆIð Ðô �‹Y˜!‹^à$˜+ Q˜ˆIØÐrG   c                óJ  • U R                   R                  U5      nU R                  U5      nU R                  X2S5      u  p4U SU R	                  U5       S3n[
        R                  R                  U5      nU[        R                  [        R                  4;   a  [        R                  nSU S3nU R                  (       GaÂ  [        U R                  [        5      (       a  U R                  R                  c   e[!        / U R#                  U5      QU R                  R                  Q75      nU R%                  U R'                  U5      5      nUR                  (       aê  U R(                  R+                  UR,                   S35        U R(                  R+                  UR,                   SU R                   S35        U R/                  U R0                  =(       d    S5      n	U R(                  R+                  U S	[3        U5       S
U	 S35        U R(                  R+                  U SU S[3        U5       S
UR,                   S35        U$ U R(                  R+                  U SU R                   S
U S[3        U5       S35        U$ U R5                  XPR#                  U5      5      $ )z"Codegen a load from an InputBufferFr0  r1  rj   r9   z!_mask = hl.RDom([hl.Range(0, 1)])z_mask.where(r   z = hl.cast(rb   rä  z + hl.cast(z_mask)z = hl.select(z
, hl.cast(z, 0)))ru   Úinputræ  r  r1  r#   rQ  Ú	get_dtyper<   rô   rõ   rö   Ú
_load_maskr:   r  r"  r   ré  Únewfuncr)  r  rƒ  r  rù  Ú_load_otherrð   rç  )
rN   r  rê  rë  r4  r  rï   r"  r  r  s
             rE   rš  ÚHalideKernel.load‚  sK  € à�i‰i�o‰o˜dÓ#ˆØ×%Ñ% eÓ,ˆØ×/Ñ/°¸EÓB‰	ˆØ��a˜×+Ñ+¨DÓ1Ð2°!Ð4ˆÜ—‘×!Ñ! $Ó'ˆØ”U—]‘]¤E§N¡NÐ3Ó3Ü—M‘MˆEØ+¨D¨6°Ð3ˆDà�?�?ˆ?ä˜4Ÿ?™?Ô,=×>Ñ>Ø—O‘O×-Ñ-Ñ9ðð:ô #ØO�$×+Ñ+¨EÓ2ÐO°T·_±_×5NÑ5NÑOóˆIð —\‘\ $×"5Ñ"5°iÓ"@ÓAˆFØ××Ø—	‘	×#Ñ# v§{¡{ mÐ3TÐ$UÔVØ—	‘	×#Ñ# v§{¡{ m°<ÀÇÁÐ?PÐPQÐ$RÔSØŸ
™
 4×#3Ñ#3×#8°qÓ9�Ø—	‘	×#Ñ#Ø�h˜k¬+°eÓ*<Ð)=¸RÀ¸wÀaÐHôð —	‘	×#Ñ#Ø�h˜c $  {´;¸uÓ3EÐ2FÀbÈÏÉÈÐU[Ð\ôð ˆMð —	‘	×#Ñ#Ø�h˜m¨D¯O©OÐ+<¸B¸t¸fÀJÌ{Ð[`ÓOaÐNbÐbgÐhôð ˆMà—<‘< ×&?Ñ&?ÀÓ&FÓGÐGrG   c                ó`   • U R                   R                  [        R                  " SSU5         $ )Nz\[.*Ú )ÚcseÚvarname_maprH  rÓ  ©rN   r  s     rE   rZ  ÚHalideKernel.lookup_cse_var©  s$   € Ø�x‰x×#Ñ#¤B§F¢F¨7°B¸Ó$=Ñ>Ð>rG   c                ó<  • [        U[        5      (       d   eU R                  R                  U5      nU R	                  U5      nU R                  XRS5      u  pVU R                  U5      (       d  Ub‡  U R                  5       nU R                  Xg5      nUR                  U5      n	SR                  S/[        U5      -  5      =(       d    Sn
U R                  R                  [        X SU
 SU S35      5        OU R                  USS	9n[        U5      n	[         R"                  R%                  U5      nUc  U SU S
['        U5       SU	 S3nO,US:X  a  U SU S['        U5       SU	 S3nO[)        SU 35      eU R                  R                  [        X5      5        g)z"Codegen a store to an OutputBufferTNrb   rV  r/  r0  z] = hl.undef(z.type()))r[  z] = hl.cast(r9   Ú
atomic_addz] += hl.cast(zstore mode=)r:   r  ru   Úoutputræ  r  Úis_indirect_indexingrß  r1  rF  r2  rt   r  rƒ  r'   rñ  r#   rQ  r5  rð   rÐ   )rN   r  rê  r  Úmoderë  r4  rE  r5  Ú	value_strÚ
undef_dimsrï   r  s                rE   ÚstoreÚHalideKernel.store¬  s‘  € ô ˜%Ô!2×3Ñ3Ð3Ð3Ø�i‰i×Ñ˜tÓ$ˆØ×%Ñ% eÓ,ˆØ×/Ñ/°¸DÓA‰	ˆØ×$Ñ$ U×+Ñ+¨tÑ/?Ø×2Ñ2Ó4ˆLØ×+Ñ+¨DÓ?ˆIØŸ™ |Ó4ˆIØŸ)™) Z L´3°t³9Ñ$<Ó=×FÀ$ˆJØ�I‰I×ÑÜ˜T U¨!¨J¨<°}ÀSÀEÈÐ#RÓSõð ×+Ñ+¨D¸DÐ+ÐAˆIÜ˜E›
ˆIä—‘×!Ñ! $Ó'ˆØ‰<Ø�U˜!˜I˜; l´;¸uÓ3EÐ2FÀbÈÈÐSTÐU‰DØ�\Ó!Ø�U˜!˜I˜; m´KÀÓ4FÐ3GÀrÈ)ÈÐTUÐV‰Dä%¨°D°6Ð&:Ó;Ð;Ø�	‰	×ÑœL¨Ó4Õ5rG   c           	     ó¼  • U R                   (       d   eU R                  (       a   eX#U4nXPR                  R                  ;   a  U R                  R                  U   $ [	        U[
        5      (       a2  US:X  d   eU R                  " U6 =U R                  R                  U'   nU$ [	        U[        5      (       a  UR                  c   e[        U R                  5      nU R                  UR                   Vs/ s H  oˆU;  d  M
  UPM     sn5      n	U[        UR                  5      -
  (       aC  U R                  U U R                  [        / UR                  QUQ75      5      UR                  S9nUR                  U R                  5      n
[         R"                  R%                  X25      n['        U5      nUS;   aÆ  U	R(                   SU 3nU R*                  R-                  U SU SU
 S35        / nS	n[/        U R                  5       HD  u  nnUR1                  U S
U S35        US	:w  a  US==   SU 3-  ss'   XðR2                  U   -  nMF     U R*                  R-                  U	 SSR5                  U5       35        O­US:X  a  U R7                  X5      n	O•[9        X<5      n[:        R<                  " [?        [A        5       5      5         U" Xš5      nSSS5        SU S[C        U5       S3nU R*                  R-                  U	 SU 35        U R*                  R-                  U	 SW 35        X�R                  R                  U'   U	$ s  snf ! , (       d  f       N= f)zCodegen a reduction operationÚwelford_combineNr!  )ÚargmaxÚargminÚ_z = hl.z(rdom, r9   r$   r0  r1  éÿÿÿÿÚ*rä  r—  Úwelford_reducera   rb   )"rq  r6  r<  Úreduction_cacher:   ÚtupleÚwelford_combine_implr  r"  r   rx  r7  rç  r)  r   rF  r   Ú	ReductionÚdefault_accumulatorr÷   r  r  rƒ  rå  r¸  rv  r2  Úwelford_reduce_fallbackr   r#   Úset_ops_handlerr   rù   rF   )rN   rï   rü   r  r  Ú	cache_keyÚresult_tupleÚreduction_varsrÒ  Ú
result_varrE  ÚdefaultÚacc_typerê  ÚpartsrN  r×  r\  Ú
combine_fnÚcombine_strÚdefault_strs                        rE   Ú	reductionÚHalideKernel.reductionÉ  s  € ð ×$×$Ð$Ð$Ø—?—?Ð"Ð"Ø°Ð6ˆ	ØŸ™×0Ñ0Ó0Ø—8‘8×+Ñ+¨IÑ6Ð6ä�eœU×#Ñ#Ø!Ð%6Ó6Ð6Ð6à×)Ò)¨5Ð1ðˆD�H‰H×$Ñ$ YÑ/°,ð  Ðä˜%Ô!2×3Ñ3¸¿¹Ñ8SÐSÐSÜ# D×$:Ñ$:Ó;ˆØ—\‘\ØŸšÓCš�1°NÑ+B�Q™ÑCó
ˆ
ð œJ u§¡Ó7×7Ø—L‘LØ�'Ø×#Ñ#¤JÐ/R°·±Ð/RÀ>Ñ/RÓ$SÓTØ—k‘kð !ð ˆEð
 —N‘N 4×#9Ñ#9Ó:ˆ	Ü—,‘,×2Ñ2°>ÓMˆÜ" 5Ó)ˆàÐ1Ó1Ø!—‘Ð' q¨Ð(8Ð9ˆEØ�I‰I×Ñ 5 '¨°Ð/?¸wÀyÀkÐQRÐ SÔTàˆEØˆFÜ# D×$:Ñ$:Ö;‘��3à—‘ ˜w a¨ s¨!˜_Ô-Ø˜Q“;à˜"“I 1 V H Ñ-“IØ×*Ñ*¨3Ñ/Ñ/’ñ <ð �I‰I×Ñ : ,¨c°%·*±*¸UÓ2CÐ1DÐ EÕFØÐ/Ó/à×5Ñ5°eÓC‰Jä1°.ÓKˆJÜ×"Ò"¤?´?Ó3DÓ#EÕFÙ(¨Ó?�÷ Gà$ X J¨b´ÀÓ1IÐ0JÈ!ÐLˆKØ�I‰I×Ñ : ,¨c°+°Ð ?Ô@Ø�I‰I×Ñ : ,¨c°+°Ð ?Ô@à.8�‰× Ñ  Ñ+ØÐùòM D÷> GÕFús   Ã6	MÄMË	MÍ
Mc                ó  • [        U[        5      (       a  UR                  c   e[        U[        5      (       a  UR                  c   e[        U[        5      (       a  UR                  c   e[        / UR                  QUR                  QUR                  Q7=(       d    U R                  5      nU[        U R
                  5      -  nU R                  U R                  U5      5      nXU4 Vs/ s H  nSUR                   S3PM     nnUR                  nU R                  R                  U SSR                  U5       S35        U R                  R                  U SU S35        U R                  R                  U SU S	35        U R                  R                  U S
U S35        U R                  R                  U SUR                  U R
                  5       35        U R                  R                  U SUR                  U R
                  5       35        U R                  R                  U SUR                  U R
                  5       35        U R                  R                  U SU SU S35        U R                  R                  U SU SU S35        U R                  R                  U SU SU SU S35        U SU SU S3U SU SU SU SU SU S3U S3/n	U R                  R                  U SSR                  U	5       S35        / n
[        S 5       HT  nU
R                  U R                  UR                  5      5        U R                  R                  U
S!    S"U S#U S$35        MV     [        U
5      $ s  snf )%Nra   z.type(), 0)z = hl.Tuple([rb   rã  z
_mean_1 = z[0]z_m2_1 = z[1]z_weight_1 = z[2]z
_mean_2 = z_m2_2 = z_weight_2 = z	_delta = z
_mean_2 - Ú_mean_1z_new_weight = z_weight_1 + Ú	_weight_2z_w2_over_w = hl.select(z_new_weight == 0.0, 0.0, z_weight_2 / z_new_weight)z
_mean_1 + z	_delta * Ú
_w2_over_wz_m2_1 + z_m2_2 + z_weight_1 * Ú_new_weightr   rN  rä  r0  r1  )r:   r  r"  r   rv  rx  r7  r)  r  r  rƒ  r2  rF  r
  r¸  rR  )rN   ÚmeanÚm2Úweightr"  r[  rÖ   r\  Úpfxr(  Úunpackedr×  s               rE   rS  Ú!HalideKernel.welford_combine_impl	  sh  € Ü˜$Ô 1×2Ñ2°t·~±~Ñ7QÐQÐQÜ˜"Ô/×0Ñ0°R·\±\Ñ5MÐMÐMÜ˜&Ô"3×4Ñ4¸×9IÑ9IÑ9UÐUÐUÜØ?ˆd�n‰nÐ?˜rŸ|™|Ð?¨f×.>Ñ.>Ñ?×SÀ4×CSÑCSó
ˆ	ð 	”Z × 6Ñ 6Ó7Ñ7ˆ	Ø—\‘\ $×"5Ñ"5°iÓ"@ÓAˆ
Ø<@ÀfÑ;MÓNÒ;M°a�X˜aŸf™f˜X [Ó1Ñ;MˆÐNØ�o‰oˆØ�	‰	×Ñ˜z˜l¨-¸¿	¹	À'Ó8JÐ7KÈ2ÐNÔOØ�	‰	×Ñ˜s˜e :¨j¨\¸Ð=Ô>Ø�	‰	×Ñ˜s˜e 8¨J¨<°sÐ;Ô<Ø�	‰	×Ñ˜s˜e <°
¨|¸3Ð?Ô@Ø�	‰	×Ñ˜s˜e :¨d¯m©m¸D×<RÑ<RÓ.SÐ-TÐUÔVØ�	‰	×Ñ˜s˜e 8¨B¯K©K¸×8NÑ8NÓ,OÐ+PÐQÔRØ�	‰	×ÑØˆe�< §¡°×0FÑ0FÓ GÐHÐIô	
ð 	�	‰	×Ñ˜s˜e 9¨S¨E°¸C¸5ÀÐHÔIØ�	‰	×Ñ˜s˜e >°#°°lÀ3À%ÀyÐQÔRØ�	‰	×ÑØˆeÐ*¨3¨%Ð/HÈÈÈ\ÐZ]ÐY^Ð^jÐkô	
ð ˆe�:˜c˜U )¨C¨5°
Ð;Øˆe�8˜C˜5 ¨¨¨Y°s°e¸9ÀSÀEÈÐVYÐUZÐZdÐeØˆe�;Ðð
ˆð
 	�	‰	×Ñ˜z˜l¨-¸¿	¹	À&Ó8IÐ7JÈ"ÐMÔNàˆÜ�q–ˆAØ�O‰O˜DŸL™L¨×)=Ñ)=Ó>Ô?Ø�I‰I×Ñ 8¨B¡< .°°J°<¸qÀÀÀ1Ð EÖFñ ô �X‹Ðùò7 Os   Ã.Nc           
     ó®  • U R                   (       d   e[        U5      [        U5      :X  d   e/ n[        [        R                     " 5       nU HÌ  n[        U[        5      (       a  UR                  c   e[        UR                  5      [        U R                  5      -  (       a  UR                  U5        OHUR                  U R                  U / UR                  Q/ U R                  QS S PUR                  S95        UR                  UR                  5        MÎ     U R                  U R                  U5      5      nUR                  (       a0  [        UR                  5      [        U R                  5      -  (       d   e[        X5       VVs/ s H  u  p†S[!        U5       SU S3PM     n	nnU R#                  U R%                  U R&                  S   R(                  5      5      n
UR*                   S3nU S3nU R,                  R/                  U S	U
 S
35        [        U R                  5      S:X  d   S5       e/ U R                  Qu  nU[1        U5      0nU[1        U5      S-
  0n[        U5      S:X  a(  S nUR3                  U5      /nUR3                  U5      /nOwS n[5        [        U5      5       Vs/ s H  nUR3                  U5      SU S3-   PM     nn[5        [        U5      5       Vs/ s H  nUR3                  U5      SU S3-   PM     nnU R,                  R/                  U SU" U	5       35        [6        R8                  " [;        [=        5       5      5         U" UU5      nS S S 5        U R,                  R/                  UR3                  U5       SU" W5       35        [        U5      S:X  a  U4$ U Vs/ s H#  nU R                  U R                  U5      5      PM%     nn[?        U5       H*  u  nnU R,                  R/                  U SU SU S35        M,     [A        U5      $ s  snnf s  snf s  snf ! , (       d  f       NÙ= fs  snf )Nr$   r!  ra   rb   r9   rN  Ú_rdomz.xz = hl.RDom([hl.Range(1, z)])z&multi-dimensional scan not implementedc                ó   • U S   $ rP  rR   r
  s    rE   Úmaybe_tupleÚ&HalideKernel.scan.<locals>.maybe_tuple`  s   € Ø˜‘t�rG   c                ó,   • SSR                  U 5       S3$ )Nz
hl.Tuple([rb   rã  )r2  r
  s    rE   rr  rs  g  s   € Ø# D§I¡I¨a£L >°Ð4Ð4rG   r0  r1  rä  )!rq  rt   r   r–   rY  r:   r  r"  rx  r¸  rç  r   r(  r7  r)  r  r÷   rù  rú  r°  r¹  r  r  rƒ  r    rF  r
  r#   rW  r   rù   rå  rR  )rN   Údtypesr_  Úvalues_origr'  Úall_used_dimsr  r[  rï   Úinitialr“  Úscan_domÚscanÚscan_varÚscan_renames_curÚscan_renames_prirr  Ú	read_leftÚ
read_rightr×  r`  rM  Úunpack_varsrÒ  s                           rE   rz  ÚHalideKernel.scan/  s1  € ð ×$×$Ð$Ð$Ü�6‹{œc +Ó.Ó.Ð.Ð.Ø*,ˆÜ"¤5§<¡<Ò0Ó2ˆã ˆEÜ˜eÔ%6×7Ñ7¸E¿O¹OÑ<WÐWÐWÜ˜%Ÿ/™/Ó*¬Z¸×8NÑ8NÓ-O×OØ—‘˜eÕ$à—‘Ø—L‘LØ ˜'ØI˜%Ÿ/™/ÐIÐ+D¨T×-CÑ-CÐ+DÀRÀaÐ+HÐIØ#Ÿk™kð !ð ôð × Ñ  §¡Ö1ñ !ð —\‘\ $×"5Ñ"5°mÓ"DÓEˆ
Ø×#×#¬
°:×3GÑ3GÓ(HÌ:Ø×"Ñ"óL
÷ )
ð 	
ð 
ô
 !$ FÔ 3ô
â 3‘�ð ” uÓ-Ð.¨b°°°qÓ9Ù 3ð 	ñ 
ð
 —‘˜D×0Ñ0°×1AÑ1AÀ"Ñ1E×1KÑ1KÓLÓMˆØ —o‘oÐ& eÐ,ˆØ�˜2ˆˆØ�	‰	×Ñ˜x˜jÐ(@ÀÀÈÐLÔMä�4×)Ñ)Ó*¨aÓ/ð 	
Ø4ó	
Ð/ð 0˜×.Ñ.Ð/‰ˆØ$Ô&8¸Ó&>Ð?ÐØ$Ô&8¸Ó&>ÀÑ&BÐCÐäˆv‹;˜!Óòð $×,Ñ,Ð-=Ó>Ð?ˆIØ$×-Ñ-Ð.>Ó?Ð@‰Jò5ô
 œs 6›{Ô+óâ+�Að ×#Ñ#Ð$4Ó5¸!¸A¸3¸a¸Ô@Ù+ð ð ô œs 6›{Ô+óâ+�Að ×#Ñ#Ð$4Ó5¸!¸A¸3¸a¸Ô@Ù+ð ð ð
 	�	‰	×Ñ˜z˜l¨#©k¸'Ó.BÐ-CÐDÔEô ×Òœ¬Ó/@ÓAÕBÙ$ Y°
Ó;ˆK÷ Cà�	‰	×ÑØ×"Ñ"Ð#3Ó4Ð5°S¹À[Ó9QÐ8RÐSô	
ô ˆv‹;˜!ÓØ�=Ð áQWÓXÒQWÈA�t—|‘| D×$7Ñ$7¸Ó$FÖGÑQWˆÐXÜ˜kÖ*‰DˆAˆqØ�I‰I×Ñ 1 # S¨¨°A°a°S¸Ð :Ö;ñ +ä�[Ó!Ð!ùók
ùò:ùò÷ CÕBüò Ys$   ÆP1Ê9"P7Ë3"P<Í$
QÏ*QÑ
Qrÿ  c               óŠ   • U R                   R                  U R                  XUS9n[        U[        5      (       d   eX%l        U$ )Nrÿ  )r<  Úgenerater  r:   r  r"  )rN   r  r"  rå  r   rë  s         rE   rç  ÚHalideKernel.genfunc„  sA   € ð �h‰h×Ñ §	¡	¨4ÀeÐÐLˆÜ˜#Ô0×1Ñ1Ð1Ð1Ø!ŒØˆ
rG   r!  c               ór   • U R                   R                  US9n[        U[        5      (       d   eXl        U$ r   )r<  Únewvarr:   r  r"  )rN   r"  r   rë  s       rE   r7  ÚHalideKernel.newfunc‘  s4   € Ø�h‰h�o‰o EˆoÐ*ˆÜ˜#Ô0×1Ñ1Ð1Ð1Ø!ŒØˆ
rG   c                óx   • [         R                  R                  U5      R                  5       R	                  5       $ )zõ
We map all tensors to 1D buffers in Halide since Halide has trouble representing some strides that PyTorch
supports.  If there are gaps in the underlying layout the numel we pass to Halide includes the gaps while
PyTorch's numel excludes them.
)r#   rQ  Ú
get_bufferÚ
get_layoutÚstorage_sizer>  s     rE   Úhalide_buffer_numelÚ HalideKernel.halide_buffer_numel—  s+   € ô �w‰w×!Ñ! $Ó'×2Ñ2Ó4×AÑAÓCÐCrG   c                ó–  ^• S n/ nU R                   R                  5       u  p4pS[        [        XE5      US9 HŽ  u  nmUR	                  UT45        [        T[        5      (       d  M0  TR                  S:X  a  TR                  b   eUR                  U4S jU R                  R                  TR                  S5       5       5        M�     U$ )zH
Halide requires scalar inputs before outputs, so need to reorder args.
c                ó~   • U u  p[        U[        5      (       a  gSUR                  ;   a  gSUR                  ;   d   eg)Nr$   Úout_ptrr   Úin_ptrr   )r:   r,   r  )Ú	arg_tupleÚ	_call_strr,  s      rE   Ú	arg_orderÚ.HalideKernel.halide_argdefs.<locals>.arg_order¤  s=   € Ø&‰NˆIÜ˜#œw×'Ñ'ØØ˜cŸh™hÓ&Øà 3§8¡8Ó+Ð+Ð+ØrG   r¦  r   c           
   3  ó’   >#   • U  H<  nS [        UTR                  TR                  TR                  TR                  S94v •  M>     g 7f)N)Úalias_of)r-   Úbufferrï   r‹  r  )r?  Úaliasr,  s     €rE   r@  Ú.HalideKernel.halide_argdefs.<locals>.<genexpr>´  sJ   øé € ð ò "G˜ð Ü!Ø!ØŸJ™JØŸI™IØŸJ™JØ%(§X¡Xñõ	ò "Gùs   ƒAArR   )ru   Úpython_argdefsr  r  r¸  r:   r-   r‹  r—  r¾  r{  rD  r  )rN   r”  r  rM  rš   r›   Úcall_strr,  s          @rE   Úhalide_argdefsÚHalideKernel.halide_argdefsŸ  s®   ø€ ò
	ð =?ˆØ—Y‘Y×-Ñ-Ó/‰
ˆˆaÜ#¤C¨£I°9Ô=‰MˆH�cØ�M‰M˜8 S˜/Ô*Ü˜#œy×)Ó)Ø—z‘z Q“¨3¯<©<Ñ+?Ð?Ð?Ø—‘ô ð "&×!4Ñ!4×!8Ñ!8¸¿¹À2Ô!Fóö ñ	 >ð" ˆrG   c                ó.  • / nU R                  5        GH=  u  p#[        U[        5      (       a	  SnSnSnSnOèU R                  UR                      Vs/ s H'  n[        U R                  UR                  5      5      PM)     nnU R                  UR                      Vs/ s H'  n[        U R                  UR                  5      5      PM)     nn[        U5      [        U5      :X  d   e[        U R                  UR                     5      n[        UR                      S3nUR                  [        UUR                  UUUUR                  S95        GM@     [         R"                  R%                  5       n	U	R&                  S:X  aE  [(        R*                  R,                  /n
[(        R*                  R.                  nS[1        5       0nSnGOU	R&                  S:X  d   S5       eU	R2                  S	::  d   S
5       e[(        R*                  R4                  /n
[(        R*                  R6                  n[8        R:                  R=                  U	5      nSU
S	   ;  aF  S H@  u  nnUR>                  U:¼  d  M  UR@                  U:¼  d  M*  U
R                  SU U 35          O   U
R                  S5        SURB                  0n[E        S	U	R2                  5      nU
R                  S5        U
R                  S5        [(        R*                  RF                  (       d  U
R                  S5        [(        R*                  RH                  (       a  U
R                  S5        SU RJ                  ;   a  U
R                  S5        [M        USRO                  U
5      UUUS9$ s  snf s  snf )z)Compute metadata required by codecache.pyNÚlongrO  )r   rN  r‹  r—  ÚcpuÚparallelismÚcudazonly cpu/cuda supportedr   zonly default device supportedÚcuda_capability))é   é   )r¥  r   )é   é   )r§  r   )r¦  r$   Úcuda_capability_Úuser_contextÚstrict_floatÚ
no_runtimeÚ
no_assertsÚdebugÚ64Úlarge_buffersÚ-)ÚtargetÚ	schedulerÚscheduler_flagsÚcuda_device)(r�  r:   r,   rt  r  r/   rú  ró  rN  rt   ru  r.   rï   r¸  r   r—  r#   rQ  Úget_current_device_or_throwÚtyper   r  Ú
cpu_targetÚscheduler_cpur   rê  Ú
gpu_targetÚscheduler_cudar<   r£  Úget_device_propertiesÚmajorÚminorÚmulti_processor_countr@   Úassertsr®  rd   r   r2  )rN   ÚargtypesrM  r,  r   rN  r‹  rï   rÖ   Úcurrent_devicer²  r³  r´  rµ  Ú
capabilityr½  r¾  s                    rE   Úhalide_kernel_metaÚHalideKernel.halide_kernel_metaÃ  s  € àˆØ×)Ñ)×+‰FˆAÜ˜#œw×'Ñ'Ø�Ø�Ø�Ø‘ð "×3Ñ3°C·H±HÒ=óâ=˜ô ˜$×.Ñ.¨q¯v©vÓ6Ö7Ù=ð ð ð "×3Ñ3°C·H±HÒ=óâ=˜ô ˜$×.Ñ.¨q¯x©xÓ8Ö9Ù=ð ð ô ˜5“z¤S¨£[Ó0Ð0Ð0Ü˜t×2Ñ2°3·8±8Ñ<Ó=�Ü'¨¯	©	Ñ2Ð3°1Ð5�Ø�O‰OÜØØ—H‘HØØ!Ø!Ø Ÿ\™\ñ÷	ñ% ,ô: Ÿ™×<Ñ<Ó>ˆØ×Ñ %Ó'Ü—m‘m×.Ñ.Ð/ˆFÜŸ™×3Ñ3ˆIàÔ3Ó5ðˆOð ŠKà!×&Ñ&¨&Ó0ÐKÐ2KÓKÐ0Ø!×'Ñ'¨1Ó,ÐMÐ.MÓMÐ,Ü—m‘m×.Ñ.Ð/ˆFÜŸ™×4Ñ4ˆIÜŸ™×9Ñ9¸.ÓIˆJØ ¨¨q©	Ó1Û$L‘L�E˜5Ø!×'Ñ'¨5Õ0°Z×5EÑ5EÈÕ5NØŸ™Ð(8¸¸À¸wÐ&GÔHÙñ %Mð �M‰M˜.Ô)à˜z×?Ñ?ðˆOô
 ˜a ×!5Ñ!5Ó6ˆKð 	�‰�nÔ%ð 	�‰�lÔ#ä�}‰}×$×$Ø�M‰M˜,Ô'ä�=‰=××Ø�M‰M˜'Ô"à�4×#Ñ#Ó#à�M‰M˜/Ô*äØØ—8‘8˜FÓ#ØØ+Ø#ñ
ð 	
ùòCùòs   Á.NÂ.Nc                óÔ
  ^ • T R                   R                  (       a  [        S5      eT R                  5       n[	        5       nUR                  SSS9  UR                  5         T R                  5        HÑ  u  pE[        U[        5      (       a-  UR                  UR                   ST R                   S35        MG  UR                  (       d   U5       eSUR                  ;   a  SOS	n[        UR                  5      n[!        T R"                  UR                     5      nUR                  UR                   S
U SU SU S35        MÓ     UR                  S5        UR                  5         T R                  5        H/  u  pEUR                  UR                   SUR                   35        M1     T R                   R%                  5        H  u  pšUR                  U	 S
U
 35        M     UR                  T R&                  5        U 4S jnT R(                  R*                   HH  n[        U[,        5      (       a  [.        R0                  R3                  X¼5      nUR                  U5        MJ     UR                  S5        UR                  S5        T R                  5        GHÌ  u  pE[        U[        5      (       aU  [4        R6                  R8                  R;                  UR<                  SS9nUR                  UR                   SU S35        Mp  T R"                  UR                     n/ n[?        U5       GH  u  nnT RA                  [4        R6                  R8                  R;                  URB                  SS9U5      nURE                  SU S35        SUR                  ;  d  Mp  UR                  UR                   SU S35         UR                  UR                   SU S[G        URH                  5       S35         UR                  UR                   SU S[G        URB                  5       S35        GM     UR                  UR                   SSRM                  U5       S35        GMÏ     URO                  S5        UR                  SRQ                  5       5        URR                  (       am  UR                  S[T        RV                  " URR                  5      < SURX                  < S URR                  < SURZ                  < S!3	SS9  UR]                  5       $ UR                  S"URX                  < S#3SS9  UR]                  5       $ ! [J         a     GN\f = f! [J         a     GM7  f = f)$z3Called at the end to generate a final kernel stringÚinplace_bufferszÒ
            import halide as hl
            from torch._inductor.runtime import halide_helpers
            from math import inf, nan

            @hl.generator(name="kernel")
            class Kernel:
        T©Ústripz = hl.InputScalar(r9   Úoutzhl.OutputBufferzhl.InputBufferrä  rÞ   rb   z&
            def generate(g):
        z = g.c                ó®   >• [        [        TR                  R                  U R	                  S5         5      nUR
                  c   U5       e[        U5      $ )Nr$   )r   r  r<  r=  rÛ  r"  rñ  )rÞ  rë  rN   s     €rE   Úupdate_indexÚ1HalideKernel.codegen_kernel.<locals>.update_index=  sE   ø€ ÜÔ(¨$¯(©(×*>Ñ*>¸q¿w¹wÀq»zÑ*JÓKˆCØ—=‘=Ñ,Ð1¨cÓ1Ð,Ü�s“8ˆOrG   r;  zassert g.using_autoscheduler()r$   r‡  z.set_estimate(râ  z.dim(z).set_min(0)z).set_stride(z).set_extent(z.set_estimates([rã  r   zN
            if __name__ == "__main__":
                hl.main()
            z:
                else:
                    hl.load_plugin(z))
                    target = hl.Target(z=)
                    autoscheduler = hl.AutoschedulerParams(a�  )
                    with hl.GeneratorContext(target, autoscheduler):
                        gen = Kernel()
                        pipeline = gen._build_pipeline()
                        # gen.compile_to_callable() does not run the autoscheduler
                        pipeline.apply_autoscheduler(target, autoscheduler)
                        kernel = pipeline.compile_to_callable([
                                gen._get_input_parameter(a.name)._to_argument()
                                for a in gen._get_arginfos()
                                if a.dir == hl.ArgInfoDirection.Input
                            ], target)
                zR
                  else:
                      with hl.GeneratorContext(hl.Target(zX)):
                          kernel = Kernel().compile_to_callable()
                  )/ru   rÇ  rI   rÄ  r(   ÚspliceÚ	do_indentr�  r:   r,   rƒ  r  rd   r˜  rð   rï   rt   rt  ÚaliasesrÃ  r  Ú_linesrñ  r  rJ  rÓ  r#   rQ  rR  r£  rf   rå  Ú_autoscheduler_workaroundsró  r¸  r;   rN  rd  r2  Údo_unindentÚrstripr³  r   Úfind_libautoscheduler²  r´  Úgetvalue)rN   r  ÚmetaÚcoderM  r,  ÚargclsÚargtypeÚndimr  r  rÌ  r  Úhintr4  Úrange_hintsr×  Údims   `                 rE   Úcodegen_kernelÚHalideKernel.codegen_kernel  s·  ø€ à�9‰9×$×$ÜÐ/Ó0Ð0Ø×&Ñ&Ó(ˆÜÓˆØ�‰ðð ð 	ñ 
	
ð 	�‰ÔØ×)Ñ)Ö+‰FˆAÜ˜#œw×'Ñ'Ø—‘ #§(¡( Ð+=¸d×>NÑ>NÐ=OÈqÐQÖRà—z—zÐ& 3Ó&�zØ.3°s·x±xÓ.?Ñ*ÐEU�Ü% c§i¡iÓ0�Ü˜4×1Ñ1°#·(±(Ñ;Ó<�Ø—‘ #§(¡( ¨3¨v¨h°a¸°yÀÀ4À&ÈÐJÖKñ ,ð 	�‰ðô	
ð
 	�‰ÔØ×)Ñ)Ö+‰FˆAØ�N‰N˜cŸh™h˜Z u¨S¯X©X¨JÐ7Ö8ñ ,àŸ	™	×)Ñ)Ö+‰HˆCØ�N‰N˜c˜U # c UÐ+Ö,ñ ,à�‰�D×&Ñ&Ô'õ	ð
 —I‘I×$Ô$ˆDÜ˜$¤×$Ñ$ä(×5Ñ5×9Ñ9¸,ÓM�Ø�N‰N˜4Ö ñ	 %ð
 	�‰�rÔØ�‰Ð7Ô8à×)Ñ)×+‰FˆAô ˜#œw×'Ñ'Ü—w‘w×'Ñ'×1Ñ1°#·(±(ÀQÐ1ÐG�Ø—‘ #§(¡( ¨>¸$¸¸qÐAÖBà×-Ñ-¨c¯h©hÑ7�Ø �Ü'¨Ÿo‘F�A�sØ×:Ñ:ÜŸ™×(Ñ(×2Ñ2°3·8±8ÀaÐ2ÐHÈ$ó�Dð  ×&Ñ&¨°d°V¸1Ð'=Ô>Ø C§H¡HÕ,ØŸ™¨#¯(©(¨°5¸¸¸<Ð'HÔIð!Ø ŸN™NØ#&§8¡8 *¨E°!°°MÄ#ÀcÇjÁjÃ/ÐARÐRSÐ Tôð
!Ø ŸN™NØ#&§8¡8 *¨E°!°°MÄ#ÀcÇhÁhÃ-ÀÐPQÐ R÷ñ .ð( —‘ #§(¡( Ð+;¸D¿I¹IÀkÓ<RÐ;SÐSUÐV×Wñ= ,ð@ 	×Ñ˜ÔØ�‰ð÷ ‘“ô		
ð �>�>Ø�K‰Kð$ä$3×$HÒ$HÈÏÉÓ$XÑ#[ð \(Ø(,¯©¡ð 7<Ø<@¿N¹NÑ;MÈRÐPT×PdÑPdÑOgð hð	ð  ð# ñ ð8 �}‰}‹Ðð �K‰Kð:à:>¿+¹+¹ð Iðð
 ð ñ ð �}‰}‹Ðøô]  )ó !Ú ð!ûô  )ó !Û ð!ús$   Ï7UÏ97UÕ
UÕUÕ
U'Õ&U'c                óÐ   • [        U5      S:X  aV  [        R                  R                  S:X  a8  [        R
                  R                  5       R                  S:X  a  [        SU 5      n U $ )Nr$   ÚAnderson2021r£  r   )	rt   r   r  r»  r#   rQ  r¶  r·  r@   )rä   r4  s     rE   rÒ  Ú'HalideKernel._autoscheduler_workarounds�  sN   € ô �‹I˜‹NÜ—‘×,Ñ,°Ó>Ü—‘×3Ñ3Ó5×:Ñ:¸fÓDô �A�q“	ˆAØˆrG   c                ó¶  • [         R                  R                  nU R                  5        VVs/ s H  u  pVUR                  b  M  U PM     nnn[         R                  R                  5       nUR                  S:X  aE  UR                  UR                  [         R                  R                  5      n	UR                  U	5        UR                  UUUSS9  gs  snnf )zCodegen a call to this kernelNr£  F)ÚdeviceÚtriton)r#   rQ  Úwrapper_coder�  r—  r¶  r·  Úwrite_get_raw_streamrê  r  r¸  Úgenerate_kernel_call)
rN   r  r—  Údeallocate_wsÚwrapperrä   r,  Ú	call_argsrÂ  Ústream_names
             rE   Úcall_kernelÚHalideKernel.call_kernel›  s¶   € ä—'‘'×&Ñ&ˆØ*.×*=Ñ*=Ô*?ÔXÒ*?¡ À3Ç<Á<“V˜�s’VÑ*?ˆ	ÑXÜŸ™×<Ñ<Ó>ˆØ×Ñ &Ó(Ø!×6Ñ6Ø×$Ñ$¤a§g¡g§l¡lóˆKð ×Ñ˜[Ô)Ø×$Ñ$ØØØ!Øð	 	%ò 	
ùó Ys   ®CÁCc                ó   • gr^  rR   )rN   rô  s     rE   Úgenerate_assertÚHalideKernel.generate_assert¬  s   € ØrG   c                ó   • g rì   rR   )rN   rf   ró  ÚlowerÚuppers        rE   Úcheck_boundsÚHalideKernel.check_bounds¯  s   € ð 	rG   )r{  rt  ru  rm  ry  rv  r|  rs  rw  rp  rn  rr  rx  ro  )r}  zdict[str, sympy.Expr]rT   rU   )rï   r  rT   rñ  )NNN)rÆ  zSequence[sympy.Expr])rê  rM  )rë  rñ  rê  rM  rü  rû   r^  )r  rñ  rê  rM  )r  rñ  rì   )
r  rñ  rê  rM  r  r&   rD  r6   rT   rU   )
rï   r  rü   r  r  r5   r  ú+Union[CSEVariable, tuple[CSEVariable, ...]]rT   rø  )ru  ztuple[torch.dtype, ...]r_  zUCallable[[tuple[CSEVariable, ...], tuple[CSEVariable, ...]], tuple[CSEVariable, ...]]rv  útuple[CSEVariable, ...]rT   rù  )r   r7   rT   r  )rT   r   r  )r  rñ  rê  rû   )rf   rM  ró  rM  rô  rû   rõ  rû   ),rV   rW   rX   rY   rù   Ú	overridesÚtexprrù  r_  rM   r€  r„  rÔ  rß  rÄ  ræ  rî  r  r  r  ré  r)  r1  rš  rZ  rG  rb  rS  rz  r   Úunknownrç  r7  rŒ  r�  rÄ  rß  rç   rÒ  rî  rñ  rö  rZ   r[   r\   s   @rE   rk  rk  Ì  s¾  ø‡ Ø€IØ).€EÐ&Ó.ð+à%ð+ð 
÷	+ô6"ô=÷
lò\ò*BðNà÷Nò%ôf:òPò(
ô.ò.
ôô%HôN?ð SWð6Øð6Ø *ð6Ø3>ð6ØFOð6à	õ6ð:>àð>ð ð>ð &ð	>ð
 ;ð>ð 
5ô>ò@$ðLS"à'ðS"ð
ðS"ð -ðS"ð 
!ôS"ðt ×"Ò"Ó$Ø $ñð ðð 
õð =A÷ ôDò"ôHQ
ôfxðt ñó ðö
ò"ðØðØ&0ðØ9=ðØFJ÷ò rG   rk  c                  ó2   • \ rS rSr\r\SS j5       rS rSr	g)ÚHalideSchedulingiµ  c                óò   • [        [        R                  [        R                  [        R                  /5      n[
        R                  R                  (       a  UR                  [        R                  5        U$ rì   )
r   r%   ÚTUPLE_REDUCTIONÚPREFER_STORE_LOOP_ORDERÚREDUCE_TO_SINGLE_ELEMENTr   r  Úscan_kernelsrŽ  ÚSCAN)r  rå  r  s      rE   Úget_backend_featuresÚ%HalideScheduling.get_backend_features¸  sR   € ää×.Ñ.Ü×6Ñ6Ü×7Ñ7ðó
ˆô �=‰=×%×%Ø�J‰J”~×*Ñ*Ô+ØˆrG   c                ó  • [         R                  R                  nXR                  ;   a  UR                  U   nU$ SUR	                  5        3nXTR                  U'   UR                  S5        [        5       nUR                  SUR                  5       < S35        UR                  USS9  UR                  S5        [        X$5      u  pxU SU 3n	UR                  XVR                  5       U	5        [        S	5      (       a  [        US
U5        U$ )z6Codegen kernel definition to go in output wrapper codeÚhalide_kernel_zEfrom torch._inductor.runtime.hints import HalideMeta, HalideInputSpeczasync_compile.halide(z, '''TrÈ  z''')Ú
Úkernel_metadatar;  )r#   rQ  rç  Úsrc_to_kernelÚnext_kernel_suffixÚadd_import_oncer(   rƒ  rÄ  rÎ  r   Údefine_kernelrÖ  r   r   )
rN   Úsrc_codeÚnode_schedulerc   rë  Úkernel_nameÚcompile_wrapperÚoriginsÚdetailed_originsÚmetadata_comments
             rE   r  ÚHalideScheduling.define_kernelÅ  s  € ä—'‘'×&Ñ&ˆØ×,Ñ,Ó,Ø!×/Ñ/°Ñ9ˆKð. Ðð+ +¨7×+EÑ+EÓ+GÐ*HÐIˆKØ.9×!Ñ! (Ñ+Ø×#Ñ#ØWôô -Ó.ˆOØ×%Ñ%Ø'¨×(AÑ(AÓ(CÑ'FÀeÐLôð ×"Ñ" 8°4Ð"Ñ8Ø×%Ñ% fÔ-ä(;¸MÓ(SÑ%ˆGØ") ¨"Ð-=Ð,>Ð?ÐØ×!Ñ!Ø×5Ñ5Ó7Ð9Iôô 'Ð'8×9Ñ9Ü# K°°XÔ>àÐrG   rR   N)rå  ztorch.devicerT   zOrderedSet[BackendFeature])
rV   rW   rX   rY   rk  Úkernel_typer  r  r  rZ   rR   rG   rE   rþ  rþ  µ  s   † Ø€Kàó
ó ð
õrG   rþ  )tÚ
__future__r   Údataclassesr«  r%  ÚloggingrH  Úcollectionsr   Úmathr   Útypingr   r   r   r	   r
   r–   r<   Útorch._loggingÚ_prims_commonr   Úutils._ordered_setr   Úutils._sympy.functionsr   r   Úutils._sympy.symbolr   r   Úutils._sympy.value_rangesr   r;  r   r   Ú	codecacher   r   Úmetricsr   r   Úops_handlerr   Úruntime.hintsr   r   Úutilsr   r   r   r    r!   Úvirtualizedr"   r™  r#   Úcommonr%   r&   r'   r(   r)   r*   r+   r,   r-   Úcppr.   Ú	cpp_utilsr/   Úsimdr0   r1   r2   Úcollections.abcr3   r4   r5   r6   Úshape_propagationr7   Ú	getLoggerrV   rµ  rF   ÚRuntimeErrorrI   r^   rŽ   rû  Úpexprrû   rõ   rô   rö   Úfloat64rÒ  Úint16ró   r>   Úuint8Úuint16Úuint32Úuint64rí   rð   r÷   rù   Ú_initialize_pointwise_overridesr  Ú	dataclassrL  rf  rÑ  rk  rþ  rR   rG   rE   Ú<module>r;     sÒ  ðå "ã Û Û Û Û 	Ý #Ý ß <Õ <ã ã Û å -Ý ,ß ?ß 7Ý 4ß Ý 'Ý )ß BÝ )ß 7÷õ ÷ )÷
÷ 
õ 
õ Ý ß ;Ñ ;ö ß2ç6Ý2à×Ò˜Ó!€ò
ôF�,ô Fô
VQ�Mô VQñr 	‹×Ñ€Ù‹×Ñ€ð 
‡J�J�Ø	‡N�N�OØ	‡M�M�>Ø	‡M�M�>Ø	‡M�M�>Ø	‡J�J�Ø	‡K�K�Ø	‡K�K�Ø	‡K�K�Ø	‡K�K�Ø	‡L�L�-Ø	‡L�L�-Ø	‡L�L�-ð€ò"òôV"�kô V"ðr
 × /Ñ /°Ô 9ô%P˜ô %PðP ×Ñ÷+ð +ó ð+ò@
òô f�:ô fôR+�~õ +rG   