ó
    pyüiÐ[  ã                  ó²  • % S r SSKJr  SSKrSSKrSSKJr  SSKJ	r	  SSK
JrJr  SSKrSSKJr  SSKJr  S	S
KJr  S	SKJr  S	SKJr  S	SKJrJr  S	SKJrJrJrJrJrJ r J!r!J"r"J#r#  \(       a  S	SK$J%r%  S	SKJ&r&  \RN                  " \(5      r)\	 " S S5      5       r*S(S jr+\\*/\,\\-\.\4   4   4   r/S)S jr0 S*               S+S jjr1S,S jr2S,S jr3S,S jr4S,S jr5S,S jr6S,S jr7S,S jr8S,S jr9S,S jr:S,S jr;S,S  jr<S,S! jr=S,S" jr>S,S# jr?S,S$ jr@S,S% jrA\R„                  \R†                  \Rˆ                  \RŠ                  \RŒ                  \RŽ                  \R�                  \R’                  \R”                  \R–                  \R˜                  \Rš                  \Rœ                  \Rž                  \R                   /rQ\R¤                  \R¦                  \R¨                  \Rª                  \R¬                  \R®                  /rX\R²                  \R´                  /r[\R¸                  \Rº                  /r^\R¾                  \RÀ                  \RÂ                  /rb\RÆ                  \2\RÈ                  \3\RÊ                  \3\RÌ                  \4\RÎ                  \5\RÐ                  \7\RÒ                  \8\RÔ                  \9\RÖ                  \:\RØ                  \>\RÚ                  \A\RÜ                  \=\RÞ                  \=0\-Rá                  \Q\65      E\-Rá                  \X\;5      E\-Rá                  \[\<5      E\-Rá                  \^\?5      E\-Rá                  \b\@5      ErqS&\rS''   g)-z,
Optimizer utilities for the Trainer class.
é    )ÚannotationsN)ÚCallable)Ú	dataclass)ÚTYPE_CHECKINGÚAny)Úversion)Únné   )Ú	Adafactor)ÚLayerWiseDummyOptimizer)Úcheck_target_module_exists)ÚOptimizerNamesÚParallelMode)	Úis_apollo_torch_availableÚis_bitsandbytes_availableÚis_galore_torch_availableÚis_grokadamw_availableÚis_lomo_availableÚis_schedulefree_availableÚis_torch_optimi_availableÚis_torchao_availableÚ	strtobool)ÚPreTrainedModel)ÚTrainingArgumentsc                  óL   • \ rS rSr% SrS\S'   S\S'   S\S'   S\S	'   S
\S'   Srg)ÚOptimizerContexté6   z0Context object passed to all optimizer handlers.r   ÚargszPreTrainedModel | NoneÚmodelúdict[str, Any]Úoptimizer_kwargsÚadam_kwargsúdict[str, str]Ú
optim_args© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__annotations__Ú__static_attributes__r%   ó    Ú[/home/mande/repo/quber/.venv/lib/python3.13/site-packages/transformers/trainer_optimizer.pyr   r   6   s#   ‡ á:à
ÓØ!Ó!Ø$Ó$ØÓØÖr-   r   c                óš   • U (       d  0 $ 0 nU R                  SS5      R                  S5       H  nUR                  S5      u  p4XAU'   M     U$ )z8Parse optimizer arguments from a comma-separated string.Ú Ú Ú,Ú=)ÚreplaceÚsplit)Úoptim_args_strr$   ÚmappingÚkeyÚvalues        r.   Ú_parse_optim_argsr:   A   sP   € æØˆ	Ø€JØ!×)Ñ)¨#¨rÓ2×8Ñ8¸Ö=ˆØ—]‘] 3Ó'‰
ˆØ�3‹ñ >ð Ðr-   c                ó‚   • [        U [        5      (       a*  [        U [        R                  R
                  5      (       a  gg)a  
Check if the returned value from a handler is a factory rather than an Optimizer class.

Factory callables are used for complex optimizers like Muon or Dion that need to:
- Split parameters between multiple internal optimizers
- Handle complex sharding logic
- Access the full model structure for parameter grouping

Args:
    optimizer_cls_or_factory: The first element returned by an optimizer handler.

Returns:
    `bool`: True if it's not an Optimizer class (i.e., likely a factory), False if it's an Optimizer class.
FT)Ú
isinstanceÚtypeÚ
issubclassÚtorchÚoptimÚ	Optimizer)Úoptimizer_cls_or_factorys    r.   Úis_optimizer_factoryrC   P   s2   € ô  Ð*¬D×1Ñ1´jÐAYÔ[`×[fÑ[f×[pÑ[p×6qÑ6qØØr-   c                ó   ^• UR                  5       R                  S5      nU(       a4  U R                  [        R                  :X  a  U(       a  [        SU S35      eX2   nU R                  c  [        SU S35      e[        U R                  [        [        45      (       d  [        SU R                   35      eUc  [        SU S35      e[        U R                  [        5      =(       a    U R                  R                  S	S
5      S:H  n	/ n
UR                  5        HŠ  u  p¼[        U R                  USS9u  pÞ[        U[        R                   5      (       d+  U(       a"  U(       d  ["        R%                  U SU S35        Mf  U(       d	  U	(       d  Mv  U
R'                  US-   5        MŒ     [)        U
5      S:X  a  [        SU SU R                   S35      eUR+                  5        VVs/ s H  u  nnXú;   d  M  UPM     nnnUR+                  5        VVs/ s H  u  nnXú;  d  M  UPM     nnnSU0SU0UE/nU(       a¯  U R,                  S:w  a  [        SU S35      e0 mU H  nU" SU/0/40 UD6TU'   M     U H  nU" SU/0UE/40 UD6TU'   M     U4S jnUR/                  5        H'  nUR0                  (       d  M  UR3                  U5        M)     [4        nUR7                  ST05        UR7                  SU05        X…4$ s  snnf s  snnf )zª
Helper function to set up low-rank optimizers like GaLore and Apollo.

These optimizers apply low-rank projections to specific target modules (typically linear layers).
Ú	layerwisezLayer-wise z" does not support DDP at this timez1You need to define `optim_target_modules` to use z optimizerszX`optim_target_modules` must be a list of strings, a regex string, or 'all-linear'. Got: z'You need to pass a model to initialize z optimizer.Ú_Ú-z
all-linearT)Úreturn_is_regexz matched but ignored. z only supports linear layers.z.weightr   zNo target modules found for z (z).Úparamsr
   z
Layerwise z( does not support gradient accumulation!c                ón   >• U R                   b'  TU    R                  5         TU    R                  5         g g )N)ÚgradÚstepÚ	zero_grad)ÚparamÚoptimizer_dicts    €r.   Úoptimizer_hookÚ1_setup_low_rank_optimizer.<locals>.optimizer_hook®   s4   ø€ Ø�z‰zÑ%Ø˜uÑ%×*Ñ*Ô,Ø˜uÑ%×/Ñ/Õ1ð &r-   rO   )ÚlowerÚendswithÚparallel_moder   ÚDISTRIBUTEDÚNotImplementedErrorÚoptim_target_modulesÚ
ValueErrorr<   ÚlistÚstrÚ	TypeErrorr4   Únamed_modulesr   r	   ÚLinearÚloggerÚwarningÚappendÚlenÚnamed_parametersÚgradient_accumulation_stepsÚ
parametersÚrequires_gradÚ"register_post_accumulate_grad_hookr   Úupdate)r   r   Úoptimizer_nameÚoptimizer_mappingÚoptim_kwargsr!   Úis_layerwise_supportedÚis_layerwiseÚoptimizer_clsÚ
all_linearÚtarget_params_namesÚmodule_nameÚmoduleÚtarget_module_existsÚis_regexÚnÚpÚtarget_paramsÚnon_target_paramsÚparam_groupsrN   rP   rO   s                         @r.   Ú_setup_low_rank_optimizerry   e   s/  ø€ ð "×'Ñ'Ó)×2Ñ2°;Ó?€LÞ˜×*Ñ*¬l×.FÑ.FÓFÖKaÜ! K°Ð/?Ð?aÐ"bÓcÐcà%Ñ5€Mà× Ñ Ñ(ÜÐLÈ^ÐL\Ð\gÐhÓiÐiä�d×/Ñ/´$¼°×=Ñ=ÜðØ×-Ñ-Ð.ð0ó
ð 	
ð
 �}ÜÐBÀ>ÐBRÐR]Ð^Ó_Ð_ô 	�4×,Ñ,¬cÓ2×r°t×7PÑ7P×7XÑ7XÐY\Ð^aÓ7bÐfrÑ7rð ð ÐØ$×2Ñ2Ö4ÑˆÜ)CØ×%Ñ% {ÀDñ*
Ñ&Ðô ˜&¤"§)¡)×,Ñ,Þ#®HÜ—‘ + Ð.DÀ^ÐDTÐTqÐrÔsÙæ#®JÙà×"Ñ" ;°Ñ#:Ö;ñ  5ô ÐÓ 1Ó$ÜÐ7¸Ð7GÀrÈ$×JcÑJcÐIdÐdfÐgÓhÐhà#(×#9Ñ#9Ô#;ÔXÒ#;™4˜1˜a¸qÑ?W—QÑ#;€MÑXØ',×'=Ñ'=Ô'?Ô`Ò'?™t˜q !À1ÑC_ŸÑ'?ÐÑ`ð 
Ð$Ð%Ø	�=Ð1 LÐ1ð€Lö
 Ø×+Ñ+¨qÓ0Ü˜z¨.Ð)9Ð9aÐbÓcÐcàˆÛ&ˆEÙ$1°H¸u¸gÐ3FÐ2GÑ$\ÐK[Ñ$\ˆN˜5Ó!ñ 'ã"ˆEÙ$1°H¸u¸gÐ3VÈÐ3VÐ2WÑ$lÐ[kÑ$lˆN˜5Ó!ñ #õ	2ð
 ×%Ñ%Ö'ˆEØ×"×"Ñ"Ø×8Ñ8¸ÖHñ (ô 0ˆØ×ÑÐ!1°>Ð BÔCà×Ñ˜X |Ð4Ô5ØÐ*Ð*ùó? YùÛ`s   Ç)LÇ9LÈL
È%L
c                ób   • U R                   R                  SSS.5        [        U R                   4$ )zGet Adafactor optimizer.F©Úscale_parameterÚrelative_step)r!   rg   r   )Úctxs    r.   Ú_get_adafactorr   Ã   s,   € à×Ñ×Ñ°EÈEÑ RÔSÜ�c×*Ñ*Ð*Ð*r-   c                óü   • SSK Jn  U R                  R                  U R                  5        U R
                  R                  [        R                  :X  a  U R                  R                  SS05        XR                  4$ )z/Get PyTorch AdamW optimizer (regular or fused).r   ©ÚAdamWÚfusedT)	Útorch.optimr‚   r!   rg   r"   r   r@   r   ÚADAMW_TORCH_FUSED©r~   r‚   s     r.   Ú_get_adamw_torchr‡   É   sZ   € å!à×Ñ×Ñ §¡Ô0Ø
‡x�x‡~�~œ×9Ñ9Ó9Ø×Ñ×#Ñ# W¨d OÔ4Ø×&Ñ&Ð&Ð&r-   c                ó¦   •  SSK Jn  U R                  R                  U R                  5        XR                  4$ ! [
         a    [        S5      ef = f)z'Get Torch XLA syncfree AdamW optimizer.r   r�   z7Trainer failed to import syncfree AdamW from torch_xla.)Útorch_xla.amp.syncfreer‚   r!   rg   r"   ÚImportErrorrX   r†   s     r.   Ú_get_adamw_torch_xlar‹   Ó   sO   € ðTÝ0à×Ñ×#Ñ# C§O¡OÔ4Ø×*Ñ*Ð*Ð*øÜó TÜÐRÓSÐSðTúó	   ‚7: ºAc                ó¦   •  SSK Jn  U R                  R                  U R                  5        XR                  4$ ! [
         a    [        S5      ef = f)zGet NPU Fused AdamW optimizer.r   )ÚNpuFusedAdamWz3Trainer failed to import FusedAdamW from torch_npu.)Útorch_npu.optimrŽ   r!   rg   r"   rŠ   rX   )r~   rŽ   s     r.   Ú_get_adamw_torch_npu_fusedr�   Þ   sO   € ðPÝ1à×Ñ×#Ñ# C§O¡OÔ4Ø×2Ñ2Ð2Ð2øÜó PÜÐNÓOÐOðPúrŒ   c                ó´  • [        5       (       d  [        S5      eSSKJnJnJn  U R                  R                  nSU;   nSU;   a  SOSnSnU R                  nS	U;   a  UnGO§S
U;   a2  UnSU R                  R                  U R                  R                  40nGOoSU;   a  UnU R                  nGOYSU;   GaR  SSKJn	  U	n[        U R                  R                  SU R                  R                  5      5      [        U R                  R                  SU R                  R                  5      5      [        U R                  R                  SS5      5      4[        U R                  R                  SS5      5      [        U R                  R                  SU R                  R                  5      5      S.nSU R                  ;   a  [!        U R                  S   5      US'   SU R                  ;   a  [!        U R                  S   5      US'   SU0n
SU;  a  XZS'   U R"                  R%                  U5        U R"                  R%                  U
5        XpR"                  4$ )z;Get bitsandbytes optimizer (AdamW, Lion, RMSprop variants).ziYou need to install `bitsandbytes` in order to use bitsandbytes optimizers: `pip install -U bitsandbytes`r   )r‚   ÚLionÚRMSpropÚpagedÚ8bité   é    NÚadamÚlionÚbetasÚrmspropÚademamix)ÚAdEMAMixÚbeta1Úbeta2Úbeta3g§èH.ÿï?Úalphag      @Úeps)rš   r¡   r¢   Út_alphaÚt_beta3Ú
optim_bitsÚis_paged)r   rŠ   Úbitsandbytes.optimr‚   r’   r“   r   r@   r"   Ú
adam_beta1Ú
adam_beta2r$   r�   ÚfloatÚgetÚadam_epsilonÚintr!   rg   )r~   r‚   r’   r“   Ú
optim_namer¦   r¥   rm   Úadditional_optim_kwargsr�   Ú
bnb_kwargss              r.   Ú_get_bitsandbytes_optimizerr±   é   s  € ä$×&Ñ&ÜØwó
ð 	
÷ 8Ñ7à—‘—‘€JØ˜*Ñ$€HØ 
Ó*‘°€JØ€MØ!Ÿo™oÐà�ÓØŠØ	�:Ó	ØˆØ#*¨S¯X©X×-@Ñ-@À#Ç(Á(×BUÑBUÐ,VÐ"WÒØ	�jÓ	 ØˆØ"%§.¡.ÒØ	�zÔ	!Ý/à ˆô �c—n‘n×(Ñ(¨°#·(±(×2EÑ2EÓFÓGÜ�c—n‘n×(Ñ(¨°#·(±(×2EÑ2EÓFÓGÜ�c—n‘n×(Ñ(¨°&Ó9Ó:ðô
 ˜3Ÿ>™>×-Ñ-¨g°sÓ;Ó<Ü˜Ÿ™×+Ñ+¨E°3·8±8×3HÑ3HÓIÓJñ#
Ðð ˜Ÿ™Ó&Ü14°S·^±^ÀIÑ5NÓ1OÐ# IÑ.Ø˜Ÿ™Ó&Ü14°S·^±^ÀIÑ5NÓ1OÐ# IÑ.à 
Ð+€JØ˜
Ó"Ø!)�:Ñà×Ñ×ÑÐ 7Ô8Ø×Ñ×Ñ 
Ô+Ø×.Ñ.Ð.Ð.r-   c                ó  •  SSK Jn  U R                  R                  U R                  5        U R                  R                  [        U R                  R                  SS5      5      [        [        U R                  R                  SS5      5      [        [        U R                  R                  SS5      5      [        [        U R                  R                  SS	5      5      S
.5        XR                  4$ ! [         a    [        S5      ef = f)z!Get AnyPrecision AdamW optimizer.r   )ÚAnyPrecisionAdamWÚuse_kahan_summationÚFalseÚmomentum_dtypeÚfloat32Úvariance_dtypeÚcompensation_buffer_dtypeÚbfloat16)r´   r¶   r¸   r¹   z4Please install https://github.com/pytorch/torchdistx)Útorchdistx.optimizersr³   r!   rg   r"   r   r$   r«   Úgetattrr?   rŠ   rX   )r~   r³   s     r.   Ú_get_adamw_anyprecisionr½     sÞ   € ðQÝ;à×Ñ×#Ñ# C§O¡OÔ4Ø×Ñ×#Ñ#ä'0°·±×1CÑ1CÐDYÐ[bÓ1cÓ'dÜ")¬%°·±×1CÑ1CÐDTÐV_Ó1`Ó"aÜ")¬%°·±×1CÑ1CÐDTÐV_Ó1`Ó"aÜ-4Ü˜3Ÿ>™>×-Ñ-Ð.IÈ:ÓVó.ñ	ô		
ð !×"6Ñ"6Ð6Ð6øÜó QÜÐOÓPÐPðQús   ‚C2C5 Ã5Dc                ó^  • U R                   R                  5       nU R                  (       ag  S H.  nX R                  ;   d  M  [        U R                  U   5      X'   M0     SU R                  ;   a#  U R                  S   R	                  5       S;   US'   [
        R                  R                  U4$ )zGet SGD optimizer.)ÚmomentumÚ	dampeningÚweight_decayÚnesterov©ÚtrueÚ1Úyes)r!   Úcopyr$   rª   rR   r?   r@   ÚSGD©r~   Úkwargsr8   s      r.   Ú_get_sgdrË   0  s‰   € à×!Ñ!×&Ñ&Ó(€FØ
‡~‡~Û<ˆCØ—n‘nÕ$Ü# C§N¡N°3Ñ$7Ó8�“ñ =ð ˜Ÿ™Ó'Ø!$§¡°
Ñ!;×!AÑ!AÓ!CÐG[Ñ![ˆF�:ÑÜ�;‰;�?‰?˜FÐ"Ð"r-   c                óø   • U R                   R                  5       nU R                  (       a4  S H.  nX R                  ;   d  M  [        U R                  U   5      X'   M0     [        R
                  R                  U4$ )zGet Adagrad optimizer.)Úlr_decayrÁ   r¢   )r!   rÇ   r$   rª   r?   r@   ÚAdagradrÉ   s      r.   Ú_get_adagradrÏ   <  s]   € à×!Ñ!×&Ñ&Ó(€FØ
‡~‡~Û6ˆCØ—n‘nÕ$Ü# C§N¡N°3Ñ$7Ó8�“ñ 7ô �;‰;×Ñ Ð&Ð&r-   c                ó^  • U R                   R                  5       nU R                  (       ag  S H.  nX R                  ;   d  M  [        U R                  U   5      X'   M0     SU R                  ;   a#  U R                  S   R	                  5       S;   US'   [
        R                  R                  U4$ )zGet RMSprop optimizer.)r¿   r¡   r¢   rÁ   ÚcenteredrÃ   )r!   rÇ   r$   rª   rR   r?   r@   r“   rÉ   s      r.   Ú_get_rmsproprÒ   F  sŒ   € à×!Ñ!×&Ñ&Ó(€FØ
‡~‡~Û?ˆCØ—n‘nÕ$Ü# C§N¡N°3Ñ$7Ó8�“ñ @ð ˜Ÿ™Ó'Ø!$§¡°
Ñ!;×!AÑ!AÓ!CÐG[Ñ![ˆF�:ÑÜ�;‰;×Ñ Ð&Ð&r-   c                ó&  • [        5       (       d  [        S5      eSSKJnJnJn  [        R                  U[        R                  U[        R                  U[        R                  U[        R                  U[        R                  U0n[        U R                  R                  SS5      5      [        U R                  R                  SS5      5      [!        U R                  R                  SS	5      5      U R                  R                  S
S5      S.n[#        U R$                  U R&                  U R$                  R(                  XEU R*                  5      u  pgU R$                  R(                  [        R                  :X  a  UR-                  SSS.5        Xg4$ )zGet GaLore optimizer.z�You need to install `galore_torch` in order to use GaLore optimizers. Install it with `pip install git+https://github.com/jiaweizzhao/GaLore`r   )ÚGaLoreAdafactorÚGaLoreAdamWÚGaLoreAdamW8bitÚranké€   Úupdate_proj_gapéÈ   Úscaleg      Ð?Ú	proj_typeÚstd)r×   rÙ   rÛ   rÜ   Fr{   )r   rŠ   Úgalore_torchrÔ   rÕ   rÖ   r   ÚGALORE_ADAMWÚGALORE_ADAMW_8BITÚGALORE_ADAFACTORÚGALORE_ADAMW_LAYERWISEÚGALORE_ADAMW_8BIT_LAYERWISEÚGALORE_ADAFACTOR_LAYERWISEr­   r$   Úpoprª   ry   r   r   r@   r!   rg   )r~   rÔ   rÕ   rÖ   ri   Úgalore_optim_kwargsrm   r!   s           r.   Ú_get_galore_optimizerrç   R  sD  € ä$×&Ñ&ÜðVó
ð 	
÷ KÑJô 	×#Ñ# [Ü×(Ñ(¨/Ü×'Ñ'¨Ü×-Ñ-¨{Ü×2Ñ2°OÜ×1Ñ1°?ðÐô �C—N‘N×&Ñ& v¨sÓ3Ó4Ü˜sŸ~™~×1Ñ1Ð2CÀSÓIÓJÜ�s—~‘~×)Ñ)¨'°4Ó8Ó9Ø—^‘^×'Ñ'¨°UÓ;ñ	Ðô '@Ø�‰�#—)‘)˜SŸX™XŸ^™^Ð->ÐUX×UiÑUió'Ñ#€Mð ‡x�x‡~�~œ×8Ñ8Ó8Ø×Ñ°EÈEÑ RÔSØÐ*Ð*r-   c           
     ó¾  • [        5       (       d  [        S5      eSSKJn  [        R
                  U[        R                  U0n[        U R                  R                  SS5      5      U R                  R                  SS5      U R                  R                  SS	5      [        U R                  R                  S
S5      5      [        U R                  R                  SS5      5      U R                  R                  SS5      S.nUR                  U R                  5        [        U R                  U R                  U R                  R                   X#U R"                  5      $ )zGet Apollo optimizer.zŒYou need to install `apollo_torch` in order to use APOLLO optimizers. Install it with `pip install git+https://github.com/zhuhanqing/APOLLO`r   )ÚAPOLLOAdamWr×   rØ   ÚprojÚrandomÚ
scale_typeÚchannelrÙ   rÚ   rÛ   ç      ð?rÜ   rÝ   )r×   rê   rì   rÙ   rÛ   rÜ   )r   rŠ   Úapollo_torchré   r   ÚAPOLLO_ADAMWÚAPOLLO_ADAMW_LAYERWISEr­   r$   rå   rª   rg   r"   ry   r   r   r@   r!   )r~   ré   ri   Úapollo_optim_kwargss       r.   Ú_get_apollo_optimizerró   s  s  € ä$×&Ñ&ÜðUó
ð 	
õ )ô 	×#Ñ# [Ü×-Ñ-¨{ðÐô �C—N‘N×&Ñ& v¨sÓ3Ó4Ø—‘×"Ñ" 6¨8Ó4Ø—n‘n×(Ñ(¨°yÓAÜ˜sŸ~™~×1Ñ1Ð2CÀSÓIÓJÜ�s—~‘~×)Ñ)¨'°3Ó7Ó8Ø—^‘^×'Ñ'¨°UÓ;ñÐð ×Ñ˜sŸ™Ô/ä$Ø�‰�#—)‘)˜SŸX™XŸ^™^Ð->ÐUX×UiÑUióð r-   c                ó  • [        5       (       d  [        S5      eU R                  c  [        S5      eSSKJnJn  SU R                  R                  ;   a  UOUnU R                  R                  SU R                  05        X0R                  4$ )zGet LOMO optimizer.zjYou need to install `lomo_optim` in order to use LOMO optimizers. Install it with `pip install lomo-optim`zMYou need to pass a `model` in order to correctly initialize a LOMO optimizer.r   )ÚAdaLomoÚLomoÚadar   )r   rŠ   r   rX   Ú
lomo_optimrõ   rö   r   r@   r!   rg   )r~   rõ   rö   rm   s       r.   Ú_get_lomo_optimizerrù   �  sz   € ä×ÑÜð7ó
ð 	
ð
 ‡y�yÑÜÐhÓiÐiç(à$¨¯©¯©Ó6‘G¸D€MØ×Ñ×Ñ ¨#¯)©)Ð 4Ô5Ø×.Ñ.Ð.Ð.r-   c                óü  • [        5       (       d  [        S5      eSSKJn  U R                  R                  [        U R                  R                  SS5      5      [        U R                  R                  SS5      5      [        U R                  R                  SS	5      5      [        U R                  R                  S
S	5      5      [        U R                  R                  SS5      5      S.5        XR                  4$ )zGet GrokAdamW optimizer.z5Please install grokadamw with `pip install grokadamw`r   )Ú	GrokAdamWÚ
alpha_initg\�Âõ(\ï?Úlambç       @Úgammagš™™™™™¹?Úgrokking_signal_decay_rateÚgradient_clippingrî   )rü   rý   rÿ   r   r  )	r   rX   Ú	grokadamwrû   r!   rg   rª   r$   r«   )r~   rû   s     r.   Ú_get_grokadamwr  ¢  sÄ   € ä!×#Ñ#ÜÐPÓQÐQå#à×Ñ×Ñä §¡× 2Ñ 2°<ÀÓ FÓGÜ˜#Ÿ.™.×,Ñ,¨V°SÓ9Ó:Ü˜3Ÿ>™>×-Ñ-¨g°sÓ;Ó<Ü*/°·±×0BÑ0BÐC_ÐadÓ0eÓ*fÜ!& s§~¡~×'9Ñ'9Ð:MÈsÓ'SÓ!Tñ	
ôð ×*Ñ*Ð*Ð*r-   c           	     ó¤  • [        5       (       aK  [        R                  " [        R                  R                  S5      5      [        R                  " S5      :  a  [        S5      e[        R                  " [        R                  R                  S5      5      [        R                  " S5      ::  a  [        S5      e[        R                  " [        R                  R                  S5      5      [        R                  " S5      :¼  a	  SS	KJnJn  OSS	K	JnJn  U R                  R                  [        R                  :X  a  UnOUnU R                  R                  U R                   R#                  S
S5      [%        U R                   R#                  SS5      5      S.5        U R                  R                  U R&                  5        X0R                  4$ )z%Get TorchAO 4-bit or 8-bit optimizer.Útorchaoz0.4.0z®You need to have `torchao>=0.4.0` in order to use torch 4-bit optimizers. Install it with `pip install torchao` or follow the instructions here: https://github.com/pytorch/aor?   z2.4z¹You need to have `torch>2.4` in order to use torch 4-bit optimizers. Install it with `pip install --upgrade torch` it is available on pipy. Otherwise, you need to install torch nightly.z0.11.0r   )Ú	AdamW4bitÚ	AdamW8bitÚ
block_sizeé   Úbf16_stochastic_roundrµ   )r  r
  )r   r   ÚparseÚ	importlibÚmetadatarŠ   Útorchao.optimr  r  Útorchao.prototype.low_bit_optimr   r@   r   ÚADAMW_TORCH_4BITr!   rg   r$   r«   r   r"   )r~   r  r  rm   s       r.   Ú_get_torchao_optimizerr  µ  sQ  € ä×!Ñ!¤W§]¢]´9×3EÑ3E×3MÑ3MÈiÓ3XÓ%YÔ\c×\iÒ\iÐjqÓ\rÓ%rÜð,ó
ð 	
ô
 ‡}‚}”Y×'Ñ'×/Ñ/°Ó8Ó9¼W¿]º]È5Ó=QÓQÜð<ó
ð 	
ô ‡}‚}”Y×'Ñ'×/Ñ/°	Ó:Ó;¼w¿}º}ÈXÓ?VÓVß6Ð6çHà
‡x�x‡~�~œ×8Ñ8Ó8Ø!‰à!ˆà×Ñ×ÑàŸ.™.×,Ñ,¨\¸3Ó?Ü%.¨s¯~©~×/AÑ/AÐBYÐ[bÓ/cÓ%dñ	
ôð ×Ñ×Ñ §¡Ô0Ø×.Ñ.Ð.Ð.r-   c           	     óR  • [        5       (       d  [        S5      eSSKJnJn  0 nSnU R
                  R                  [        R                  :X  a2  [        S5      (       d  [        S5      eSSKJ	n  UnU R                  nSnOmU R
                  R                  [        R                  :X  a  UnU R                  nO6U R
                  R                  [        R                  :X  a  UnO[        S	5      eU R
                  R                  US
'   U(       a  U R
                  R                  US'   UR!                  [#        U R$                  R'                  SS5      5      [#        U R$                  R'                  SS5      5      S.5        U R(                  R!                  U5        X`R(                  4$ )zGet ScheduleFree optimizer.zwYou need to install `schedulefree` in order to use schedulefree optimizers. Install it with `pip install schedulefree.`r   )ÚAdamWScheduleFreeÚSGDScheduleFreeTz1.4.0z‚You need to install `schedulefree>=1.4.0` in order to use RAdamScheduleFree optimizer. Install it with `pip install schedulefree.`)ÚRAdamScheduleFreeFzInvalid schedulefree optimizerrÁ   Úwarmup_stepsÚweight_lr_powerrþ   Úrg        )r  r  )r   rŠ   Úschedulefreer  r  r   r@   r   ÚSCHEDULE_FREE_RADAMr  r"   ÚSCHEDULE_FREE_ADAMWÚSCHEDULE_FREE_SGDrX   rÁ   r  rg   rª   r$   r«   r!   )r~   r  r  r¯   Úrequire_warmupr  rm   s          r.   Ú_get_schedule_free_optimizerr  Ø  s_  € ä$×&Ñ&Üð:ó
ð 	
÷ @à ÐØ€Nà
‡x�x‡~�~œ×;Ñ;Ó;Ü(¨×1Ñ1Üð>óð õ 	3à)ˆØ"%§/¡/ÐØ‰Ø	�‰�‰œ>×=Ñ=Ó	=Ø)ˆØ"%§/¡/ÑØ	�‰�‰œ>×;Ñ;Ó	;Ø'‰äÐ9Ó:Ð:à.1¯h©h×.CÑ.CÐ˜NÑ+ÞØ25·(±(×2GÑ2GÐ Ñ/Ø×"Ñ"ä$ S§^¡^×%7Ñ%7Ð8IÈ3Ó%OÓPÜ�s—~‘~×)Ñ)¨#¨sÓ3Ó4ñ	
ôð ×Ñ×ÑÐ 7Ô8Ø×.Ñ.Ð.Ð.r-   c                ó  • [        5       (       d  [        S5      eSSKJn  U R                  R                  SS5      nUb  [        U5      nU R                  R                  SS5      nUb  [        U5      nU R                  R                  U R                  S'   [        U R                  R                  SS	5      5      UUS
.nU R                  R                  U R                  5        U R                  R                  U5        XR                  4$ )z,Get StableAdamW optimizer from torch-optimi.zwYou need to install `torch-optimi` in order to use stable_adamw optimizers. Install it with `pip install torch-optimi`.r   )ÚStableAdamWÚmax_lrNÚ	kahan_sumrÁ   Údecouple_lrF)r#  r!  r"  )r   rŠ   Úoptimir   r$   rå   rª   Úboolr   rÁ   r"   r!   rg   )r~   r   r!  r"  Ústable_adamw_kwargss        r.   Ú_get_stable_adamwr'    sé   € ä$×&Ñ&Üð:ó
ð 	
õ #à�^‰^×Ñ ¨$Ó/€FØÑÜ�v“ˆà—‘×"Ñ" ;°Ó5€IØÑÜ˜“Oˆ	à&)§h¡h×&;Ñ&;€C‡O�O�NÑ#ä˜CŸN™N×.Ñ.¨}¸eÓDÓEØØñÐð ×Ñ×Ñ §¡Ô0Ø×Ñ×ÑÐ 3Ô4Ø×,Ñ,Ð,Ð,r-   zdict[str, OptimizerHandler]Ú_OPTIMIZER_HANDLERS)r6   z
str | NoneÚreturnr#   )rB   r   r)  r%  )T)r   r   r   r   rh   rZ   ri   r    rj   r    r!   r    rk   r%  r)  útuple[Any, dict[str, Any]])r~   r   r)  r*  )sr*   Ú
__future__r   Úimportlib.metadatar  ÚloggingÚcollections.abcr   Údataclassesr   Útypingr   r   r?   Ú	packagingr   r	   Úoptimizationr   Útrainer_pt_utilsr   Útrainer_utilsr   Útraining_argsr   r   Úutilsr   r   r   r   r   r   r   r   r   Úmodeling_utilsr   r   Ú	getLoggerr&   r^   r   r:   ÚtupleÚdictrZ   ÚOptimizerHandlerrC   ry   r   r‡   r‹   r�   r±   r½   rË   rÏ   rÒ   rç   ró   rù   r  r  r  r'  Ú	ADAMW_BNBÚ
ADAMW_8BITÚPAGED_ADAMWÚPAGED_ADAMW_8BITÚADEMAMIXÚADEMAMIX_8BITÚPAGED_ADEMAMIXÚPAGED_ADEMAMIX_8BITÚLIONÚ	LION_8BITÚ
PAGED_LIONÚPAGED_LION_8BITÚRMSPROP_BNBÚRMSPROP_8BITÚRMSPROP_32BITÚ_BITSANDBYTES_OPTIMIZERSrß   rà   rá   râ   rã   rä   Ú_GALORE_OPTIMIZERSrð   rñ   Ú_APOLLO_OPTIMIZERSr  ÚADAMW_TORCH_8BITÚ_TORCHAO_OPTIMIZERSr  r  r  Ú_SCHEDULE_FREE_OPTIMIZERSÚ	ADAFACTORÚADAMW_TORCHr…   ÚADAMW_TORCH_XLAÚADAMW_TORCH_NPU_FUSEDÚADAMW_ANYPRECISIONrÈ   ÚADAGRADÚRMSPROPÚ	GROKADAMWÚSTABLE_ADAMWÚLOMOÚADALOMOÚfromkeysr(  r+   r%   r-   r.   Ú<module>r]     sª  ðòõ #ã Û Ý $Ý !ß %ã Ý Ý å #Ý 5Ý 5ß 7÷
÷ 
õ 
ö Ý/Ý0à	×	Ò	˜8Ó	$€ð ÷ð ó ðôð Ð-Ð.°°c¸4ÀÀSÀ¹>Ð6IÑ0JÐJÑKÐ ôð8 $(ðV+Ø
ðV+àðV+ð ðV+ð &ð	V+ð
 !ðV+ð %ðV+ð !ðV+ð  õV+ô|+ô'ôTôPô//ôdQô*	#ô'ô	'ô+ôBô:/ô$+ô& /ôF)/ôX-ðD ×ÑØ×ÑØ×ÑØ×#Ñ#Ø×ÑØ× Ñ Ø×!Ñ!Ø×&Ñ&Ø×ÑØ×ÑØ×ÑØ×"Ñ"Ø×ÑØ×ÑØ× Ñ ðÐ ð& ×ÑØ×$Ñ$Ø×#Ñ#Ø×)Ñ)Ø×.Ñ.Ø×-Ñ-ðÐ ð ×ÑØ×)Ñ)ðÐ ð ×#Ñ#Ø×#Ñ#ðÐ ð ×&Ñ&Ø×&Ñ&Ø×$Ñ$ðÐ ð ×Ñ˜nØ×ÑÐ 0Ø×$Ñ$Ð&6Ø×"Ñ"Ð$8Ø×(Ñ(Ð*DØ×%Ñ%Ð'>Ø×Ñ˜Ø×Ñ˜LØ×Ñ˜LØ×Ñ˜nØ×ÑÐ!2Ø×ÑÐ,Ø×ÑÐ/ð4ð ‡m�mÐ,Ð.IÓJð4ð ‡m�mÐ&Ð(=Ó>ð4ð  ‡m�mÐ&Ð(=Ó>ð!4ð" ‡m�mÐ'Ð)?Ó@ð#4ð$ ‡m�mÐ-Ð/KÓLð%4Ð Ð0ô r-   