ó
    EñiÞ  ã                   óÄ   • S SK r S SKrSSKJrJr  SSKJrJrJrJ	r	  \	" 5       (       a  S SK
Js  Jr  S SKJr  S r " S S\R$                  R&                  5      rS\4S	 jrg)
é    Né   )ÚAcceleratorStateÚGradientState)ÚDistributedTypeÚ
honor_typeÚis_lomo_availableÚis_torch_xla_availablec                 ó~  ^• [        U [        [        45      (       a  [        U U4S jU  5       5      $ [        U [        5      (       a?  [        U 5      " U R                  5        VVs0 s H  u  p#U[        UT5      _M     snn5      $ [        U [        R                  5      (       a  U R                  T5      $ U $ s  snnf )Nc              3   ó<   >#   • U  H  n[        UT5      v •  M     g 7f©N)Úmove_to_device)Ú.0ÚtÚdevices     €ÚQ/home/mande/repo/quber/.venv/lib/python3.13/site-packages/accelerate/optimizer.pyÚ	<genexpr>Ú!move_to_device.<locals>.<genexpr>   s   øé € Ð!KÂUÀ¤.°°F×";Ð";ÂUùs   ƒ)Ú
isinstanceÚlistÚtupler   ÚdictÚtypeÚitemsr   ÚtorchÚTensorÚto)Ústater   ÚkÚvs    `  r   r   r      s‘   ø€ Ü�%œ$¤˜×'Ñ'Ü˜%Ô!KÁUÓ!KÓLÐLÜ	�Eœ4×	 Ñ	 Ü�EŒ{ÀUÇ[Á[Ä]ÔSÂ]¹T¸Q˜Aœ~¨a°Ó8Ò8Á]ÒSÓTÐTÜ	�Eœ5Ÿ<™<×	(Ñ	(Ø�x‰x˜ÓÐØ€Lùó Ts   Á&B9
c                   ó  • \ rS rSrSrSS jr\S 5       r\R                  S 5       r\S 5       r	\	R                  S 5       r	\S	 5       r
\
R                  S
 5       r
S rS rS rSS jrS rS rSS jrS r\S 5       rS rS rSrg)ÚAcceleratedOptimizeré&   a~  
Internal wrapper around a torch optimizer.

Conditionally will perform `step` and `zero_grad` if gradients should be synchronized when performing gradient
accumulation.

Args:
    optimizer (`torch.optim.optimizer.Optimizer`):
        The optimizer to wrap.
    device_placement (`bool`, *optional*, defaults to `True`):
        Whether or not the optimizer should handle device placement. If so, it will place the state dictionary of
        `optimizer` on the right device.
    scaler (`torch.amp.GradScaler` or `torch.cuda.amp.GradScaler`, *optional*):
        The scaler to use in the step function if training with mixed precision.
Nc                 óv  • Xl         X0l        [        5       U l        [	        5       U l        X l        SU l        U R                  bF  SU l        U R                   R                  U l
        [        X R                   R                  5      U l        U(       a¨  U R                   R                  5       nU R                  R                  [        R                   :X  a+  ["        R$                  " X@R                  R&                  5        O[)        X@R                  R&                  5      nU R                   R+                  U5        g g ©NF)Ú	optimizerÚscalerr   Úaccelerator_stater   Úgradient_stateÚdevice_placementÚ_is_overflowÚ_accelerate_step_calledÚstepÚ_optimizer_original_step_methodÚpatch_optimizer_stepÚ_optimizer_patched_step_methodÚ
state_dictÚdistributed_typer   ÚXLAÚxmÚsend_cpu_data_to_devicer   r   Úload_state_dict)Úselfr%   r)   r&   r0   s        r   Ú__init__ÚAcceleratedOptimizer.__init__7   sâ   € Ø"ŒØŒÜ!1Ó!3ˆÔÜ+›oˆÔØ 0ÔØ!ˆÔà�;‰;Ñ"Ø+0ˆDÔ(Ø37·>±>×3FÑ3FˆDÔ0Ü2FÀtÏ^É^×M`ÑM`Ó2aˆDÔ/ö ØŸ™×2Ñ2Ó4ˆJØ×%Ñ%×6Ñ6¼/×:MÑ:MÓMÜ×*Ò*¨:×7MÑ7M×7TÑ7TÕUä+¨J×8NÑ8N×8UÑ8UÓV�
Ø�N‰N×*Ñ*¨:Õ6ð ó    c                 ó.   • U R                   R                  $ r   ©r%   r   ©r6   s    r   r   ÚAcceleratedOptimizer.stateM   s   € à�~‰~×#Ñ#Ð#r9   c                 ó$   • XR                   l        g r   r;   ©r6   r   s     r   r   r=   Q   s   € à$�‰Õr9   c                 ó.   • U R                   R                  $ r   ©r%   Úparam_groupsr<   s    r   rB   Ú!AcceleratedOptimizer.param_groupsU   s   € à�~‰~×*Ñ*Ð*r9   c                 ó$   • XR                   l        g r   rA   )r6   rB   s     r   rB   rC   Y   s   € à&2�‰Õ#r9   c                 ó.   • U R                   R                  $ r   ©r%   Údefaultsr<   s    r   rG   ÚAcceleratedOptimizer.defaults]   s   € à�~‰~×&Ñ&Ð&r9   c                 ó$   • XR                   l        g r   rF   )r6   rG   s     r   rG   rH   a   s   € à"*�‰Õr9   c                 ó:   • U R                   R                  U5        g r   )r%   Úadd_param_group)r6   Úparam_groups     r   rK   Ú$AcceleratedOptimizer.add_param_groupe   s   € Ø�‰×&Ñ& {Õ3r9   c                 ó   • U R                   R                  [        R                  :X  a;  U R                  (       a*  [
        R                  " XR                   R                  5        U R                  R                  U5        g r   )
r'   r1   r   r2   r)   r3   r4   r   r%   r5   )r6   r0   s     r   r5   Ú$AcceleratedOptimizer.load_state_dicth   sP   € Ø×!Ñ!×2Ñ2´o×6IÑ6IÓIÈd×Nc×NcÜ×&Ò& z×3IÑ3I×3PÑ3PÔQØ�‰×&Ñ& zÕ2r9   c                 ó6   • U R                   R                  5       $ r   )r%   r0   r<   s    r   r0   ÚAcceleratedOptimizer.state_dictm   s   € Ø�~‰~×(Ñ(Ó*Ð*r9   c                 óF  • U R                   R                  (       a†  S[        R                  " U R                  R
                  5      R                  ;   nU(       a  Uc  SnU R                  R                  US9  g Ub  [        S5      eU R                  R                  5         g g )NÚset_to_noneT)rS   zJ`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer.)r(   Úsync_gradientsÚinspectÚ	signaturer%   Ú	zero_gradÚ
parametersÚ
ValueError)r6   rS   Ú
accept_args      r   rW   ÚAcceleratedOptimizer.zero_gradp   s€   € Ø×Ñ×-×-Ø&¬'×*;Ò*;¸D¿N¹N×<TÑ<TÓ*U×*`Ñ*`Ñ`ˆJÞØÑ&Ø"&�KØ—‘×(Ñ(°[Ð(ÒAàÑ*Ü$Ð%qÓrÐrØ—‘×(Ñ(Õ*ð .r9   c                 óâ  • [        U R                  S5      (       a?  [        U R                  R                  5      (       a  U R                  R                  5         g[        U R                  S5      (       az  [        U R                  R                  S5      (       aT  [        U R                  R                  R                  5      (       a%  U R                  R                  R                  5         gggg)zP
Sets the optimizer to "train" mode. Useful for optimizers like `schedule_free`
Útrainr%   N)Úhasattrr%   Úcallabler]   r<   s    r   r]   ÚAcceleratedOptimizer.train|   s£   € ô �4—>‘> 7×+Ñ+´¸¿¹×9MÑ9M×0NÑ0NØ�N‰N× Ñ Õ"ä�D—N‘N K×0Ñ0Ü˜Ÿ™×0Ñ0°'×:Ñ:Ü˜Ÿ™×1Ñ1×7Ñ7×8Ñ8ð �N‰N×$Ñ$×*Ñ*Õ,ð 9ð ;ð 1r9   c                 óº   • [        U R                  S5      (       a@  [        U R                  R                  5      (       a  U R                  R                  5         ggg)zO
Sets the optimizer to "eval" mode. Useful for optimizers like `schedule_free`
ÚevalN)r^   r%   r_   rb   r<   s    r   rb   ÚAcceleratedOptimizer.evalŠ   sB   € ô �4—>‘> 6×*Ñ*¬x¸¿¹×8KÑ8K×/LÑ/LØ�N‰N×ÑÕ!ð 0MÐ*r9   c                 ó  • [        5       (       a  SSKJnJn  U R                  R
                  (       d…  U R                  R                  [        R                  :X  a]  [        R                  " U R                  5      n[        R                  " SUS[        R                  " 5       -  S9  SU R                  l        [        5       (       a  [!        U R                  WW45      (       a  g U R                  R"                  (       aÆ  U R$                  bž  U R&                  U R                  l        U R$                  R)                  U R                  U5        U R$                  R+                  5         U R,                  (       d  SU l        OSU l        U R0                  U R                  l        SU l        OU R                  R)                  U5        U R                  R                  [        R                  :X  a  SU R                  l        g g )Nr   )ÚAdaLomoÚLomoÚsumg      ð?)ÚscaleTF)r   Ú
lomo_optimre   rf   r(   Úis_xla_gradients_syncedr'   r1   r   r2   r3   Ú_fetch_gradientsr%   Ú
all_reduceÚxrÚ
world_sizer   rT   r&   r/   r,   Úupdater+   r*   r-   )r6   Úclosurere   rf   Ú	gradientss        r   r,   ÚAcceleratedOptimizer.step‘   s]  € Ü×Ñß0ð ×#Ñ#×;×;Ø×&Ñ&×7Ñ7¼?×;NÑ;NÓNä×+Ò+¨D¯N©NÓ;ˆIÜ�MŠM˜% °#¼¿º»Ñ2GÒHØ:>ˆD×ÑÔ7ä×Ñä˜$Ÿ.™.¨4°¨/×:Ñ:Øà×Ñ×-×-Ø�{‰{Ñ&Ø&*×&IÑ&I�—‘Ô#à—‘× Ñ  §¡°Ô9Ø—‘×"Ñ"Ô$à×3×3à(,�DÕ%à(-�DÔ%à&*×&JÑ&J�—‘Ô#à/4�Õ,à—‘×#Ñ# GÔ,Ø×!Ñ!×2Ñ2´o×6IÑ6IÓIØ:?ˆD×ÑÕ7ð Jr9   c                 ó–   • U R                   R                   H*  nUS    Vs/ s H  o1R                  X35      PM     snUS'   M,     g s  snf )NÚparams)r%   rB   Úget)r6   Úparameters_maprL   Úps       r   Ú_switch_parametersÚ'AcceleratedOptimizer._switch_parameters·   sC   € ØŸ>™>×6Ô6ˆKØGRÐS[ÒG\Ó$]ÒG\À!×%7Ñ%7¸Ö%=ÑG\Ñ$]ˆK˜Ó!ò 7ùÚ$]s   ¡Ac                 ó   • U R                   $ )z.Whether or not the optimizer step was skipped.)r*   r<   s    r   Ústep_was_skippedÚ%AcceleratedOptimizer.step_was_skipped»   s   € ð × Ñ Ð r9   c                 ó‚   • / SQnU R                   R                  5        VVs0 s H  u  p#X!;  d  M  X#_M     snn$ s  snnf )N)r+   r-   r/   )Ú__dict__r   )r6   Ú_ignored_keysr   r   s       r   Ú__getstate__Ú!AcceleratedOptimizer.__getstate__À   s=   € ò
ˆð
 "&§¡×!4Ñ!4Ô!6ÔQÒ!6™˜¸!Ñ:P“�’Ñ!6ÒQÐQùÓQs   ¢;±;c                 óâ   • U R                   R                  U5        U R                  bG  SU l        U R                  R
                  U l        [        X R                  R
                  5      U l        g g r$   )	r~   ro   r&   r+   r%   r,   r-   r.   r/   r?   s     r   Ú__setstate__Ú!AcceleratedOptimizer.__setstate__È   sT   € Ø�‰×Ñ˜UÔ#Ø�;‰;Ñ"Ø+0ˆDÔ(Ø37·>±>×3FÑ3FˆDÔ0Ü2FÀtÏ^É^×M`ÑM`Ó2aˆDÕ/ð #r9   )	r+   r*   r-   r/   r'   r)   r(   r%   r&   )TNr   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r7   Úpropertyr   ÚsetterrB   rG   rK   r5   r0   rW   r]   rb   r,   rx   r{   r€   rƒ   Ú__static_attributes__© r9   r   r!   r!   &   sÝ   † ñô 7ð, ñ$ó ð$ð ‡\�\ñ%ó ð%ð ñ+ó ð+ð ×Ññ3ó ð3ð ñ'ó ð'ð ‡_�_ñ+ó ð+ò4ò3ò
+ô
+ò-ò"ô$@òL^ð ñ!ó ð!òRõbr9   r!   Úaccelerated_optimizerc                 ó   ^ ^• U U4S jnU$ )Nc                  ó"   >• STl         T" U 0 UD6$ )NT)r+   )ÚargsÚkwargsrŽ   Úmethods     €€r   Úpatched_stepÚ*patch_optimizer_step.<locals>.patched_stepÑ   s   ø€ Ø8<ÐÔ5Ù�tÐ&˜vÑ&Ð&r9   r�   )rŽ   r“   r”   s   `` r   r.   r.   Ð   s   ù€ ö'ð Ðr9   )rU   r   r   r   r   Úutilsr   r   r   r	   Útorch_xla.core.xla_modelÚcoreÚ	xla_modelr3   Útorch_xla.runtimeÚruntimerm   r   ÚoptimÚ	Optimizerr!   r.   r�   r9   r   Ú<module>rž      sW   ðó ã ç 2ß YÓ Yñ ×Ñß)Ð)Ý"òôgb˜5Ÿ;™;×0Ñ0ô gbðTÐ0Dõ r9   