ó
    Eñi6—  ã                   ó
  • S SK r S SKrS SKrS SKrS SKrS SKJr  S SKJr  S SK	J
r
  S SKrSSKJr  SSKJrJrJrJrJrJrJrJrJrJrJr  SSKJr  SS	KJrJrJr  SS
K J!r!  SSK"J#r#  SSK$J%r%J&r&  / 4S jr'S r(S\ RR                  S\*\+\+4   4S jr,S\ RR                  S\-\.\+   \*\+\+4   4   4S jr/S\ RR                  S\*\+\+4   4S jr0S\ RR                  S\*S\-\.\+   \*\+\+4   4   4S jr1S\ RR                  S\-\.\+   \*\+\+4   4   4S jr2 S!S\ RR                  S\*\+\+4   S\3S\-\ RR                  \*\+\+4   4   4S jjr4S\.\+   S\*\+\+4   4S jr5S\S\ RR                  S\-\ RR                  \*\+\
4   4   4S jr6S r7 " S S 5      r8g)"é    N)Úliteral_eval)Úwhich)ÚAnyé   )ÚSageMakerConfig)ÚDynamoBackendÚPrecisionTypeÚis_fp8_availableÚis_hpu_availableÚis_mlu_availableÚis_musa_availableÚis_neuron_availableÚis_npu_availableÚis_sdaa_availableÚis_torch_xla_availableÚis_xpu_available)ÚDEEPSPEED_MULTINODE_LAUNCHERS)Úget_free_portÚis_port_in_useÚmerge_dicts)Úcompare_versionsé   )Úparse_flag_from_env)ÚDistributedTypeÚSageMakerDistributedTypec                 óÆ   • UR                  U5      u  p4[        U 5      R                  5        H0  u  pVU[        U5      R                  5       ;   d  M$  [	        X5U5        M2     U$ )z,
Filters out all `accelerate` specific args
)Úparse_known_argsÚvarsÚitemsÚkeysÚsetattr)ÚargsÚparserÚdefault_argsÚnew_argsÚ_ÚkeyÚvalues          ÚT/home/mande/repo/quber/.venv/lib/python3.13/site-packages/accelerate/utils/launch.pyÚ_filter_argsr*   /   sU   € ð ×)Ñ)¨,Ó7�K€HÜ˜4“j×&Ñ&Ö(‰
ˆØ”$�x“.×%Ñ%Ó'Õ'Ü�H 5Ö)ñ )ð €Oó    c                  óê   • S V s/ s H  n [        U 5      (       d  M  U PM     nn [        U5      S:X  a  [        S5      eUS   n[        R                  " US/5      nSU;   a  USSSS	4$ US
SSS4$ s  sn f )zñ
Determines the executable and argument names for mpirun, based on the type of install. The supported MPI programs
are: OpenMPI, Intel MPI, or MVAPICH.

Returns: Program name and arg names for hostfile, num processes, and processes per node
)ÚmpirunÚmpiexecr   z\mpirun or mpiexec were not found. Ensure that Intel MPI, Open MPI, or MVAPICH are installed.z	--versions   Open MPIú
--hostfilez-nz
--npernodez	--bind-toz-fz-ppnÚ )r   ÚlenÚOSErrorÚ
subprocessÚcheck_output)ÚxÚmpi_appsÚmpi_appÚmpirun_versions       r)   Ú_get_mpirun_argsr9   :   s‰   € ñ 1Ó=Ò0�a´E¸!·H—Ñ0€HÐ=ä
ˆ8ƒ}˜ÓÜÐtÓuÐuð �q‰k€GÜ×,Ò,¨g°{Ð-CÓD€Nà�nÓ$Ø˜ d¨L¸+ÐEÐEð ˜˜d F¨BÐ.Ð.ùò >s
   …A0�A0r"   Úcurrent_envc                 óP  • Sn[        U 5       H”  nUR                  S5      (       d  M  [        X5      nUc  M+  US:X  a;  [        US   5      XS-   '   [        US   5      XS-   '   [        US   5      XS	-   '   Ml  [        [        X5      5      X UR	                  5        3'   M–     U$ )
z&
Setup the FP8 environment variables.
ÚACCELERATE_Úfp8_Úfp8_override_linear_precisionr   ÚFP8_OVERRIDE_FPROPr   ÚFP8_OVERRIDE_DGRADr   ÚFP8_OVERRIDE_WGRAD)r   Ú
startswithÚgetattrÚstrÚupper)r"   r:   ÚprefixÚargr(   s        r)   Úsetup_fp8_envrH   R   sª   € ð €FÜ�DŽzˆØ�>‰>˜&×!Ó!Ü˜DÓ&ˆEØÓ ØÐ9Ó9ÜADÀUÈ1ÁXÃ�KÐ)=Ñ =Ñ>ÜADÀUÈ1ÁXÃ�KÐ)=Ñ =Ñ>ÜADÀUÈ1ÁXÃ�KÐ)=Ñ =Ó>ä<?ÄÈÓ@RÓ<S�K (¨3¯9©9«;¨-Ð 8Ó9ñ ð Ðr+   Úreturnc                 ó@
  • / nU R                   (       a  U R                  (       a  [        S5      e[        U SS5      nU R                  nU R
                  bo  [        5       u  pEpgn[        U SS5      n	U(       a  U(       a  [        X#-  5      OSn
UUUU R
                  UU
/-  nU(       a  X[        U5      /-  nU(       a  XU	/-  nU R                   (       dA  UR                  [        R                  5        U R                  (       a  UR                  S5        UR                  U R                  5        UR                  U R                  5        [        R                  R!                  5       n[        U R"                  =(       d    U R$                  5      US'   U R&                  (       a  S	US
'   U R(                  S:w  aõ  U R(                  bè  [+        5       (       a  U R(                  US'   OÉ[-        5       (       a  U R(                  US'   Oª[/        5       (       a  U R(                  US'   O‹[1        5       (       a  U R(                  US'   Ol[3        5       (       a  U R(                  US'   OM[5        5       (       a  U R(                  US'   O.[7        5       (       a  U R(                  US'   OU R(                  US'   US:”  a(  U R8                  c   S5       eU R:                  c   S5       eUb  US:”  d  US:”  aE  U R8                  b  U R8                  OSUS'   U R:                  b  [        U R:                  5      OSUS'   [=        US   S5      (       a  SUS'   [        S5      US'    [?        U R@                  RC                  5       5      n[        U5      US"'   U R@                  RC                  5       S#:X  a%  [G        5       (       d  [I        S$5      e[K        X5      n [M        U RN                  RQ                  5       5      nURR                  US&'   U RT                  US''   [        U RV                  5      US('   [        U RX                  5      US)'   [        U RZ                  5      US*'   [        U R\                  5      US+'   U R^                  (       a  SUS,'   X4$ ! [         a>    [        SU R@                  RC                  5        S [>        RD                  " 5        S!35      ef = f! [         a>    [        S%U RN                  RQ                  5        S [L        RD                  " 5        S!35      ef = f)-zr
Prepares and returns the command list and an environment with the correct simple launcher environment variables.
ú0--module and --no_python cannot be used togetherÚnum_processesNzbind-toÚsocketÚ1z-mÚACCELERATE_USE_CPUÚtrueÚACCELERATE_DEBUG_MODEÚallÚZE_AFFINITY_MASKÚMLU_VISIBLE_DEVICESÚSDAA_VISIBLE_DEVICESÚMUSA_VISIBLE_DEVICESÚASCEND_RT_VISIBLE_DEVICESÚHABANA_VISIBLE_MODULESÚNEURON_RT_VISIBLE_CORESÚCUDA_VISIBLE_DEVICESr   zFWhen using multiple machines, you need to specify the main process IP.zHWhen using multiple machines, you need to specify the main process port.z	127.0.0.1ÚMASTER_ADDRÚ29500ÚMASTER_PORTFzgranularity=fine,compact,1,0ÚKMP_AFFINITYÚKMP_BLOCKTIMEúUnknown mixed_precision mode: ú. Choose between Ú.ÚACCELERATE_MIXED_PRECISIONÚfp8úrFP8 is not available on this machine. Please ensure that either Transformer Engine, MSAMP or torchao is installed.úUnknown dynamo backend: ÚACCELERATE_DYNAMO_BACKENDÚACCELERATE_DYNAMO_MODEÚACCELERATE_DYNAMO_USE_FULLGRAPHÚACCELERATE_DYNAMO_USE_DYNAMICÚ*ACCELERATE_DYNAMO_USE_REGIONAL_COMPILATIONÚOMP_NUM_THREADSÚACCELERATE_CPU_AFFINITY)0Ú	no_pythonÚmoduleÚ
ValueErrorrC   Únum_machinesÚmpirun_hostfiler9   rD   ÚappendÚsysÚ
executableÚtraining_scriptÚextendÚtraining_script_argsÚosÚenvironÚcopyÚcpuÚuse_cpuÚdebugÚgpu_idsr   r   r   r   r   r   r   Úmain_process_ipÚmain_process_portr   r	   Úmixed_precisionÚlowerÚlistr
   ÚRuntimeErrorrH   r   Údynamo_backendrE   r(   Údynamo_modeÚdynamo_use_fullgraphÚdynamo_use_dynamicÚdynamo_use_regional_compilationÚnum_cpu_threads_per_processÚenable_cpu_affinity)r"   ÚcmdrL   rq   Úmpi_app_nameÚhostfile_argÚnum_proc_argÚproc_per_node_argÚbind_to_argÚbind_toÚnproc_per_noder:   r‚   r†   s                 r)   Úprepare_simple_launcher_cmd_envr•   d   s   € ð €CØ‡~‡~˜$Ÿ+Ÿ+ÜÐKÓLÐLä˜D /°4Ó8€MØ×$Ñ$€LØ×ÑÑ'ÜScÓSeÑPˆ LÀ[Ü˜$ 	¨8Ó4ˆÞ?LÖQ]œ˜]Ñ:Ô;ÐcfˆØØØØ× Ñ ØØð
ñ 	
ˆö Ø¤# mÓ"4Ð5Ñ5ˆCÞØ Ð)Ñ)ˆCØ�>�>Ø�
‰
”3—>‘>Ô"Ø�;�;Ø�J‰J�tÔØ‡J�Jˆt×#Ñ#Ô$Ø‡J�Jˆt×(Ñ(Ô)ä—*‘*—/‘/Ó#€KÜ(+¨D¯H©H×,D¸¿¹Ó(E€KÐ$Ñ%Ø‡z‡zØ/5ˆÐ+Ñ,Ø‡|�|�uÓ §¡Ñ!9Ü×ÑØ.2¯l©lˆKÐ*Ò+Ü×ÑØ15·±ˆKÐ-Ò.Ü× Ñ Ø26·,±,ˆKÐ.Ò/Ü× Ñ Ø26·,±,ˆKÐ.Ò/Ü×ÑØ7;·|±|ˆKÐ3Ò4Ü×ÑØ48·L±LˆKÐ0Ò1Ü ×"Ñ"Ø59·\±\ˆKÐ1Ò2à26·,±,ˆKÐ.Ñ/Ø�aÓØ×#Ñ#Ñ/ð 	
ØTó	
Ð/ð ×%Ñ%Ñ1ð 	
ØVó	
Ð1ð 	Ñ! m°aÓ&7¸LÈ1Ó<LØ=A×=QÑ=QÑ=] T×%9Ò%9Ðcnˆ�MÑ"ØDH×DZÑDZÑDf¤S¨×)?Ñ)?Ô%@Ðlsˆ�MÑ"Ü˜;Ð';Ñ<¸e×DÑDØ&Dˆ�NÑ#Ü'*¨1£vˆ�OÑ$ð
Ü'¨×(<Ñ(<×(BÑ(BÓ(DÓEˆô 14°OÓ0D€KÐ,Ñ-Ø×Ñ×!Ñ!Ó# uÓ,Ü×!Ñ!Üð Eóð ô $ DÓ6ˆð
Ü& t×':Ñ':×'@Ñ'@Ó'BÓCˆð
 0>×/CÑ/C€KÐ+Ñ,Ø,0×,<Ñ,<€KÐ(Ñ)Ü58¸×9RÑ9RÓ5S€KÐ1Ñ2Ü36°t×7NÑ7NÓ3O€KÐ/Ñ0Ü@CÀD×DhÑDhÓ@i€KÐ<Ñ=ä%(¨×)IÑ)IÓ%J€KÐ!Ñ"Ø××Ø14ˆÐ-Ñ.ØÐÐøô9 ó 
ÜØ,¨T×-AÑ-A×-GÑ-GÓ-IÐ,JÐJ[Ô\i×\nÒ\nÓ\pÐ[qÐqrÐsó
ð 	
ð
ûô ó 
ÜØ& t×':Ñ':×'@Ñ'@Ó'BÐ&CÐCTÔUb×UgÒUgÓUiÐTjÐjkÐló
ð 	
ð
ús   Í#R
 Ï#S Ò
ASÓATc                 óÜ  • U R                   S:X  a  [        5       U l         OU R                   c  SU l         U R                  nU R                  nU R                  nU R                   nUS:”  a|  [        X-  5      U l        [        U5      U l        [        U R                  5      U l
        [        U SS5      (       a!  [        U5      U l        [        U5      U l        O0U SU 3U l        O#[        U5      U l        Ub  [        U5      U l        US:*  =(       d    [        U R                  5      S:H  nU(       aG  [        U5      (       a7  US::  a"  SU l        ["        R$                  " S	U S
35        O['        SU S35      eU R(                  (       a  U R*                  (       a  [-        S5      eU R(                  (       a  SU l        OU R*                  (       a  SU l        [.        R0                  R3                  5       nU R4                  (       a  SUS'   [        U SS5      nUS:w  a�  U R6                  b�  [9        5       (       a  XvS'   O|[;        5       (       a  XvS'   Oh[=        5       (       a  XvS'   OT[?        5       (       a  XvS'   O@[A        5       (       a  XvS'   O,[C        5       (       a  XvS'   O[E        5       (       a  XvS'   OXvS'   U RF                  RI                  5       n [K        U5      n[        U5      US'   U RF                  RI                  5       S:X  a%  [O        5       (       d  [Q        S5      e[S        X5      n [U        U RV                  RY                  5       5      n	U	RZ                  US!'   U R\                  US"'   [        U R^                  5      US#'   [        U R`                  5      US$'   [        U Rb                  5      US%'   U Rd                  (       GaQ  SUS&'   U Rf                  (       a  U Rh                  (       d  [-        S'5      e[k        U S(5      (       a  [        U Rl                  5      OS)US*'   [        U Rn                  5      US+'   [        U Rp                  5      RI                  5       US,'   [        U Rr                  5      RI                  5       US-'   [        U Rt                  5      US.'   U Rv                  b  [        U Rv                  5      US/'   U Rx                  b  [        U Rx                  5      US0'   U Rz                  b  [        U Rz                  5      US1'   U R|                  b  [        U R|                  5      US2'   [        U R~                  5      RI                  5       US3'   [        U R€                  5      RI                  5       US4'   [        U Rf                  5      RI                  5       US5'   [        U Rh                  5      RI                  5       US6'   [        U R‚                  5      RI                  5       US7'   [        U S8S5      b  [        U R„                  5      US9'   U R†                  (       Gað  S:n
SUS;'   [        U Rˆ                  5      XjS<-   '   [        U RŠ                  5      XjS=-   '   U RŒ                  b  [        U RŒ                  5      XjS>-   '   U RŽ                  b  [        U RŽ                  5      XjS?-   '   U R�                  b  [        U R�                  5      XjS@-   '   U R’                  b  [        U R’                  5      XjSA-   '   U R”                  b  [        U R”                  5      XjSB-   '   U R–                  b  [        U R–                  5      XjSC-   '   U R˜                  b  [        U R˜                  5      XjSD-   '   [        U Rš                  5      XjSE-   '   [        U Rœ                  5      XjSF-   '   U Rž                  b  [        U Rž                  5      XjSG-   '   U R                   b  [        U R                   5      XjSH-   '   U R¢                  b  [        U R¢                  5      XjSI-   '   U R¤                  b  [        U R¤                  5      XjSJ-   '   U R¦                  b  [        U R¦                  5      XjSK-   '   U R¨                  b  [        U R¨                  5      XjSL-   '   U Rª                  b  [        U Rª                  5      XjSM-   '   U R¬                  b  [        U R¬                  5      XjSN-   '   U R®                  b  [        U R®                  5      XjSO-   '   U R°                  b  [        U R°                  5      XjSP-   '   U R²                  b  [        U R²                  5      XjSQ-   '   U R´                  b  [        U R´                  5      XjSR-   '   U R¶                  b  [        U R¶                  5      XjSS-   '   U R¸                  b  [        U R¸                  5      XjST-   '   U Rº                  b  [        U Rº                  5      XjSU-   '   U R¼                  b  [        U R¼                  5      XjSV-   '   [        U R¾                  5      USW'   U RÀ                  (       a  S)USX'   U RÂ                  (       a  [Å        X5      nU$ ! [,         a&    [-        SU S[J        RL                  " 5        S35      ef = f! [,         a>    [-        S U RV                  RY                  5        S[T        RL                  " 5        S35      ef = f)YzW
Prepares and returns an environment with the correct multi-GPU environment variables.
r   Né<s  r   Úsame_networkFÚ:TúPort `á…  ` is already in use. Accelerate will attempt to launch in a standalone-like mode by finding an open port automatically for this session. If this current attempt fails, or for more control in future runs, please specify a different port (e.g., `--main_process_port <your_chosen_port>`) or use `--main_process_port 0` for automatic selection in your launch command or Accelerate config file.ú3Tried to launch distributed communication on port `á$  `, but another process is utilizing it. Please specify a different port (such as using the `--main_process_port` flag or specifying a different `main_process_port` in your config file) and rerun your script. To automatically use the next open port (on a single node), you can set this to `0`.rK   rP   rQ   r   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   ÚACCELERATE_USE_FSDPzWWhen using `--fsdp_cpu_ram_efficient_loading` set `--fsdp_sync_module_states` to `True`Úfsdp_versionrN   ÚFSDP_VERSIONÚFSDP_SHARDING_STRATEGYÚFSDP_RESHARD_AFTER_FORWARDÚFSDP_OFFLOAD_PARAMSÚFSDP_MIN_NUM_PARAMSÚFSDP_AUTO_WRAP_POLICYÚFSDP_TRANSFORMER_CLS_TO_WRAPÚFSDP_BACKWARD_PREFETCHÚFSDP_STATE_DICT_TYPEÚFSDP_FORWARD_PREFETCHÚFSDP_USE_ORIG_PARAMSÚFSDP_CPU_RAM_EFFICIENT_LOADINGÚFSDP_SYNC_MODULE_STATESÚFSDP_ACTIVATION_CHECKPOINTINGÚfsdp_ignored_modulesÚFSDP_IGNORED_MODULESÚMEGATRON_LM_ÚACCELERATE_USE_MEGATRON_LMÚ	TP_DEGREEÚUSE_CUSTOM_FSDPÚNO_LOAD_OPTIMÚEOD_MASK_LOSSÚNO_SAVE_OPTIMÚOPTIMIZER_CPU_OFFLOADÚUSE_PRECISION_AWARE_OPTIMIZERÚOVERLAP_CPU_OPTIMIZER_D2H_H2DÚ DECODER_LAST_PIPELINE_NUM_LAYERSÚ	PP_DEGREEÚGRADIENT_CLIPPINGÚNUM_MICRO_BATCHESÚSEQUENCE_PARALLELISMÚRECOMPUTE_ACTIVATIONSÚUSE_DISTRIBUTED_OPTIMIZERÚRECOMPUTE_GRANULARITYÚRECOMPUTE_METHODÚRECOMPUTE_NUM_LAYERSÚATTENTION_BACKENDÚEXPERT_MODEL_PARALLEL_SIZEÚCONTEXT_PARALLEL_SIZEÚATTENTION_DROPOUTÚHIDDEN_DROPOUTÚATTENTION_SOFTMAX_IN_FP32ÚEXPERT_TENSOR_PARALLEL_SIZEÚCALCULATE_PER_TOKEN_LOSSÚUSE_ROTARY_POSITION_EMBEDDINGSrl   rm   )cr�   r   rL   rq   r€   rD   r”   ÚnnodesÚintÚmachine_rankÚ	node_rankrC   Úmaster_addrÚmaster_portÚrdzv_endpointr   Ú
standaloneÚwarningsÚwarnÚConnectionErrorro   rn   rp   ry   rz   r{   r~   r   r   r   r   r   r   r   r   r‚   rƒ   r	   r„   r
   r…   rH   r   r†   rE   r(   r‡   rˆ   r‰   rŠ   Úuse_fsdpÚfsdp_cpu_ram_efficient_loadingÚfsdp_sync_module_statesÚhasattrrŸ   Úfsdp_sharding_strategyÚfsdp_reshard_after_forwardÚfsdp_offload_paramsÚfsdp_min_num_paramsÚfsdp_auto_wrap_policyÚ"fsdp_transformer_layer_cls_to_wrapÚfsdp_backward_prefetchÚfsdp_state_dict_typeÚfsdp_forward_prefetchÚfsdp_use_orig_paramsÚfsdp_activation_checkpointingr®   Úuse_megatron_lmÚmegatron_lm_tp_degreeÚmegatron_lm_use_custom_fsdpÚmegatron_lm_no_load_optimÚmegatron_lm_eod_mask_lossÚmegatron_lm_no_save_optimÚ!megatron_lm_optimizer_cpu_offloadÚ)megatron_lm_use_precision_aware_optimizerÚ)megatron_lm_overlap_cpu_optimizer_d2h_h2dÚ,megatron_lm_decoder_last_pipeline_num_layersÚmegatron_lm_pp_degreeÚmegatron_lm_gradient_clippingÚmegatron_lm_num_micro_batchesÚ megatron_lm_sequence_parallelismÚ!megatron_lm_recompute_activationsÚ%megatron_lm_use_distributed_optimizerÚ!megatron_lm_recompute_granularityÚmegatron_lm_recompute_methodÚ megatron_lm_recompute_num_layersÚmegatron_lm_attention_backendÚ&megatron_lm_expert_model_parallel_sizeÚ!megatron_lm_context_parallel_sizeÚmegatron_lm_attention_dropoutÚmegatron_lm_hidden_dropoutÚ%megatron_lm_attention_softmax_in_fp32Ú'megatron_lm_expert_tensor_parallel_sizeÚ$megatron_lm_calculate_per_token_lossÚ*megatron_lm_use_rotary_position_embeddingsr‹   rŒ   Úuse_parallelism_configÚ%prepare_extend_env_parallelism_config)r"   rL   rq   r€   r�   Úneed_port_checkr:   r   r‚   r†   rF   s              r)   Úprepare_multi_gpu_envr  É   s 
  € ð
 ×Ñ Ó"Ü!.£ˆÕà	×	Ñ	Ñ	'Ø!&ˆÔà×&Ñ&€MØ×$Ñ$€LØ×*Ñ*€OØ×.Ñ.ÐØ�aÓÜ! -Ñ"?Ó@ˆÔÜ˜,Ó'ˆŒÜ˜T×.Ñ.Ó/ˆŒÜ�4˜¨×/Ñ/Ü" ?Ó3ˆDÔÜ"Ð#4Ó5ˆDÕà$3Ð#4°AÐ6GÐ5HÐ!IˆDÕä! -Ó0ˆÔØÑ(Ü"Ð#4Ó5ˆDÔð # aÑ'×F¬3¨t×/@Ñ/@Ó+AÀQÑ+F€OÞœ>Ð*;×<Ñ<Ø˜1ÓØ"ˆDŒOÜ�MŠMØÐ*Ð+ð ,Dð Dõô "ØEÐFWÐEXð Yð óð ð ‡{‡{�t—~—~ÜÐKÓLÐLØ	��Øˆ�Ø	��ØˆŒä—*‘*—/‘/Ó#€KØ‡z‡zØ/5ˆÐ+Ñ,Ü�d˜I uÓ-€GØ�%Ó˜DŸL™LÑ4Ü×ÑØ.5Ð*Ò+Ü×ÑØ18Ð-Ò.Ü× Ñ Ø29Ð.Ò/Ü× Ñ Ø29Ð.Ò/Ü×ÑØ7>Ð3Ò4Ü×ÑØ4;Ð0Ò1Ü ×"Ñ"Ø5<Ð1Ò2à29Ð.Ñ/Ø×*Ñ*×0Ñ0Ó2€OðuÜ'¨Ó8ˆô 14°OÓ0D€KÐ,Ñ-Ø×Ñ×!Ñ!Ó# uÓ,Ü×!Ñ!Üð Eóð ô $ DÓ6ˆð
Ü& t×':Ñ':×'@Ñ'@Ó'BÓCˆð
 0>×/CÑ/C€KÐ+Ñ,Ø,0×,<Ñ,<€KÐ(Ñ)Ü58¸×9RÑ9RÓ5S€KÐ1Ñ2Ü36°t×7NÑ7NÓ3O€KÐ/Ñ0Ü@CÀD×DhÑDhÓ@i€KÐ<Ñ=à‡}‡}€}Ø-3ˆÐ)Ñ*Ø×.×.°t×7S×7SÜÐvÓwÐwä@GÈÈn×@]Ñ@]¤c¨$×*;Ñ*;Ô&<Ðcfˆ�NÑ#ô 14°D×4OÑ4OÓ0PˆÐ,Ñ-ä47¸×8WÑ8WÓ4X×4^Ñ4^Ó4`ˆÐ0Ñ1Ü-0°×1IÑ1IÓ-J×-PÑ-PÓ-RˆÐ)Ñ*Ü-0°×1IÑ1IÓ-JˆÐ)Ñ*Ø×%Ñ%Ñ1Ü36°t×7QÑ7QÓ3RˆKÐ/Ñ0Ø×2Ñ2Ñ>Ü:=¸d×>eÑ>eÓ:fˆKÐ6Ñ7Ø×&Ñ&Ñ2Ü47¸×8SÑ8SÓ4TˆKÐ0Ñ1Ø×$Ñ$Ñ0Ü25°d×6OÑ6OÓ2PˆKÐ.Ñ/Ü/2°4×3MÑ3MÓ/N×/TÑ/TÓ/VˆÐ+Ñ,Ü.1°$×2KÑ2KÓ.L×.RÑ.RÓ.TˆÐ*Ñ+Ü8;¸D×<_Ñ<_Ó8`×8fÑ8fÓ8hˆÐ4Ñ5Ü14°T×5QÑ5QÓ1R×1XÑ1XÓ1ZˆÐ-Ñ.Ü7:¸4×;]Ñ;]Ó7^×7dÑ7dÓ7fˆÐ3Ñ4Ü�4Ð/°Ó6ÑBÜ25°d×6OÑ6OÓ2PˆKÐ.Ñ/à××ÐØˆØ4:ˆÐ0Ñ1Ü,/°×0JÑ0JÓ,Kˆ˜[Ñ(Ñ)Ü25°d×6VÑ6VÓ2WˆÐ.Ñ.Ñ/Ø×)Ñ)Ñ5Ü47¸×8VÑ8VÓ4WˆK Ñ0Ñ1Ø×)Ñ)Ñ5Ü47¸×8VÑ8VÓ4WˆK Ñ0Ñ1Ø×)Ñ)Ñ5Ü47¸×8VÑ8VÓ4WˆK Ñ0Ñ1Ø×1Ñ1Ñ=Ü<?À×@fÑ@fÓ<gˆKÐ!8Ñ8Ñ9Ø×9Ñ9ÑEÜDGÈ×HvÑHvÓDwˆKÐ!@Ñ@ÑAØ×9Ñ9ÑEÜDGÈ×HvÑHvÓDwˆKÐ!@Ñ@ÑAØ×<Ñ<ÑHÜGJØ×AÑAóHˆKÐ!CÑCÑDô -0°×0JÑ0JÓ,Kˆ˜[Ñ(Ñ)Ü47¸×8ZÑ8ZÓ4[ˆÐ0Ñ0Ñ1Ø×-Ñ-Ñ9Ü8;¸D×<^Ñ<^Ó8_ˆKÐ!4Ñ4Ñ5Ø×0Ñ0Ñ<Ü;>¸t×?dÑ?dÓ;eˆKÐ!7Ñ7Ñ8Ø×1Ñ1Ñ=Ü<?À×@fÑ@fÓ<gˆKÐ!8Ñ8Ñ9Ø×5Ñ5ÑAÜ@CÀD×DnÑDnÓ@oˆKÐ!<Ñ<Ñ=Ø×1Ñ1Ñ=Ü<?À×@fÑ@fÓ<gˆKÐ!8Ñ8Ñ9Ø×,Ñ,Ñ8Ü7:¸4×;\Ñ;\Ó7]ˆKÐ!3Ñ3Ñ4Ø×0Ñ0Ñ<Ü;>¸t×?dÑ?dÓ;eˆKÐ!7Ñ7Ñ8Ø×-Ñ-Ñ9Ü8;¸D×<^Ñ<^Ó8_ˆKÐ!4Ñ4Ñ5Ø×6Ñ6ÑBÜADÀT×EpÑEpÓAqˆKÐ!=Ñ=Ñ>Ø×1Ñ1Ñ=Ü<?À×@fÑ@fÓ<gˆKÐ!8Ñ8Ñ9Ø×-Ñ-Ñ9Ü8;¸D×<^Ñ<^Ó8_ˆKÐ!4Ñ4Ñ5Ø×*Ñ*Ñ6Ü58¸×9XÑ9XÓ5YˆKÐ!1Ñ1Ñ2Ø×5Ñ5ÑAÜ@CÀD×DnÑDnÓ@oˆKÐ!<Ñ<Ñ=Ø×7Ñ7ÑCÜBEÀd×FrÑFrÓBsˆKÐ!>Ñ>Ñ?Ø×4Ñ4Ñ@Ü?BÀ4×ClÑClÓ?mˆKÐ!;Ñ;Ñ<Ø×:Ñ:ÑFÜEHØ×?Ñ?óFˆKÐ!AÑAÑBô &)¨×)IÑ)IÓ%J€KÐ!Ñ"Ø××Ø14ˆÐ-Ñ.à×"×"Ü;¸DÓNˆàÐøôm ó uÜÐ9¸/Ð9JÐJ[Ô\i×\nÒ\nÓ\pÐ[qÐqrÐsÓtÐtðuûô ó 
ÜØ& t×':Ñ':×'@Ñ'@Ó'BÐ&CÐCTÔUb×UgÒUgÓUiÐTjÐjkÐló
ð 	
ð
ús   Ëi0 Ì6#j# é00j ê#Ak+c                 ó�  • SnSUS'   [        U R                  5      XS-   '   [        U R                  5      XS-   '   [        U R                  5      XS-   '   [        U R                  5      XS-   '   [        U R
                  5      XS-   '   [        U R                  5      XS	-   '   [        U R                  5      XS
-   '   U R                  S:”  a  [        U R                  5      XS-   '   U R                  S:”  aN  [        U R                  5      XS-   '   [        U R                  5      XS-   '   [        U R                  5      XS-   '   U$ )zN
Extends `current_env` with context parallelism env vars if any have been set
ÚPARALLELISM_CONFIG_rP   Ú!ACCELERATE_USE_PARALLELISM_CONFIGÚDP_REPLICATE_SIZEÚDP_SHARD_SIZEÚTP_SIZEÚCP_SIZEÚ
CP_BACKENDÚSP_SIZEÚ
SP_BACKENDr   ÚCP_COMM_STRATEGYÚSP_SEQ_LENGTHÚSP_SEQ_LENGTH_IS_VARIABLEÚSP_ATTN_IMPLEMENTATION)rD   Ú$parallelism_config_dp_replicate_sizeÚ parallelism_config_dp_shard_sizeÚparallelism_config_tp_sizeÚparallelism_config_cp_sizeÚparallelism_config_cp_backendÚparallelism_config_sp_sizeÚparallelism_config_sp_backendÚ#parallelism_config_cp_comm_strategyÚ parallelism_config_sp_seq_lengthÚ,parallelism_config_sp_seq_length_is_variableÚ)parallelism_config_sp_attn_implementation)r"   r:   rF   s      r)   r  r  ’  sB  € ð #€Fà7=€KÐ3Ñ4Ü03°D×4]Ñ4]Ó0^€KÐ,Ñ,Ñ-Ü,/°×0UÑ0UÓ,V€K˜Ñ(Ñ)Ü&)¨$×*IÑ*IÓ&J€K˜Ñ"Ñ#Ü&)¨$×*IÑ*IÓ&J€K˜Ñ"Ñ#Ü),¨T×-OÑ-OÓ)P€K˜Ñ%Ñ&Ü&)¨$×*IÑ*IÓ&J€K˜Ñ"Ñ#Ü),¨T×-OÑ-OÓ)P€K˜Ñ%Ñ&Ø×&Ñ&¨Ó*Ü36°t×7_Ñ7_Ó3`ˆÐ/Ñ/Ñ0Ø×&Ñ&¨Ó*Ü03°D×4YÑ4YÓ0Zˆ˜_Ñ,Ñ-Ü<?À×@qÑ@qÓ<rˆÐ8Ñ8Ñ9Ü9<¸T×=kÑ=kÓ9lˆÐ5Ñ5Ñ6àÐr+   c                 ó0  • U R                   S:X  a  [        5       U l         OU R                   c  SU l         U R                  nU R                  nU R                  nU R                   nSnU R
                  c  [        S   U l        US:”  Ga@  U R
                  [        S   :w  Ga(  S/nUR                  S[        U R                  5      /5        U R
                  S:X  aE  [        SSS	5      (       a  [        S
5      eUR                  S[        U R                  5      S/5        O'UR                  SS[        U R
                  5      /5        U R                  b'  UR                  S[        U R                  5      /5        OgU R                  b'  UR                  S[        U R                  5      /5        O3UR                  S[        U R                  U R                  -  5      /5        U(       a  UR                  S[        U5      /5        UR                  S[        U5      /5        U R                  (       a  U R                   (       a  [        S5      eU R                  (       a  UR#                  S5        O"U R                   (       a  UR#                  S5        UR#                  U R$                  5        UR                  U R&                  5        O¼US:”  a“  U R
                  [        S   :X  a|  [        X-  5      U l        [        U5      U l        [-        U R                  5      U l        [1        U SS5      (       a!  [        U5      U l        [        U5      U l        O0U SU 3U l        O#[        U5      U l        Ub  [        U5      U l        US:*  =(       d    [-        U R                  5      S:H  nU(       aG  [9        U5      (       a7  US::  a"  SU l        [<        R>                  " SU S35        O[A        SU S35      eU R                  (       a  U R                   (       a  [        S5      eU R                  (       a  SU l        OU R                   (       a  SU l        [B        RD                  RG                  5       nU RH                  (       a  SUS '   [1        U S!S"5      nUS":w  a�  U RJ                  b�  [M        5       (       a  X‡S#'   O|[O        5       (       a  X‡S$'   Oh[Q        5       (       a  X‡S%'   OT[S        5       (       a  X‡S&'   O@[U        5       (       a  X‡S''   O,[W        5       (       a  X‡S('   O[Y        5       (       a  X‡S)'   OX‡S*'    [[        U R\                  R_                  5       5      n	[c        S.[B        Rd                  Rg                  S-5      5      US.'   [        U	5      US/'   U R\                  R_                  5       S0:X  a%  [i        5       (       d  [k        S15      e[m        X5      n[        U Rn                  5      R_                  5       US2'   SUS3'   U Rp                  b  [        U Rp                  5      US4'   U Rr                  b  [        U Rr                  5      US5'   U Rt                  b&  [        U Rt                  5      R_                  5       US6'   U Rv                  b&  [        U Rv                  5      R_                  5       US7'   U Rx                  b&  [        U Rx                  5      R_                  5       US8'   U Rz                  b&  [        U Rz                  5      R_                  5       US9'   U R|                  b&  [        U R|                  5      R_                  5       US:'   U R~                  b  [        U R~                  5      US;'   U R€                  (       a  S<US='   U R‚                  b  [        U R‚                  5      US>'   U R„                  (       a  [‡        X5      nXW4$ ! [         a>    [        S+U R\                  R_                  5        S,[Z        R`                  " 5        S-35      ef = f)?zl
Prepares and returns the command list and an environment with the correct DeepSpeed environment variables.
r   Nr—   r   Ú	deepspeedr/   ÚnosshÚ<z0.14.5z+nossh launcher requires DeepSpeed >= 0.14.5z--node_rankz--no_sshz--no_local_rankz
--launcherz	--excludez	--includez
--num_gpusz--master_addrz--master_portrK   z--modulez--no_pythonr˜   Fr™   Trš   r›   rœ   r�   rP   rQ   r   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r`   ra   rb   Ú
PYTHONPATHrc   rd   re   ÚACCELERATE_CONFIG_DS_FIELDSÚACCELERATE_USE_DEEPSPEEDÚACCELERATE_DEEPSPEED_ZERO_STAGEÚ&ACCELERATE_GRADIENT_ACCUMULATION_STEPSÚACCELERATE_GRADIENT_CLIPPINGÚ-ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICEÚ)ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICEÚACCELERATE_DEEPSPEED_ZERO3_INITÚ+ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODELÚ ACCELERATE_DEEPSPEED_CONFIG_FILErN   rm   Ú(ACCELERATE_DEEPSPEED_MOE_LAYER_CLS_NAMES)Dr�   r   rL   rq   r€   Údeepspeed_multinode_launcherr   rw   rD   Údeepspeed_hostfiler   rp   rÏ   Údeepspeed_exclusion_filterÚdeepspeed_inclusion_filterro   rn   rs   rv   rx   r”   rÍ   rÎ   rÐ   rC   rÑ   rÒ   rÓ   r   rÔ   rÕ   rÖ   r×   ry   rz   r{   r~   r   r   r   r   r   r   r   r   r	   r‚   rƒ   r„   Úenv_var_path_addÚpathÚabspathr
   r…   rH   Ú'deepspeed_fields_from_accelerate_configÚ
zero_stageÚgradient_accumulation_stepsÚgradient_clippingÚoffload_optimizer_deviceÚoffload_param_deviceÚzero3_init_flagÚzero3_save_16bit_modelÚdeepspeed_config_filerŒ   Údeepspeed_moe_layer_cls_namesr  r  )
r"   rL   rq   r€   r�   r�   r  r:   r   r‚   s
             r)   Úprepare_deepspeed_cmd_envrA  ­  sú  € ð
 ×Ñ Ó"Ü!.£ˆÕà	×	Ñ	Ñ	'Ø!&ˆÔà×&Ñ&€MØ×$Ñ$€LØ×*Ñ*€OØ×.Ñ.ÐØ
€Cð ×(Ñ(Ñ0ä,IÈ!Ñ,LˆÔ)à�aÔ˜D×=Ñ=ÔA^Ð_`ÑAaÔaØˆmˆØ�
‰
�L¤# d×&=Ñ&=Ó">Ð?Ô@Ø×,Ñ,°Ó7Ü ¨S°(×;Ñ;Ü Ð!NÓOÐOØ�J‰J˜¤s¨4×+<Ñ+<Ó'=¸zÐJÕKà�J‰JÐ)¨<¼¸T×=^Ñ=^Ó9_Ð`ÔaØ×*Ñ*Ñ6Ø�J‰JàÜ˜×7Ñ7Ó8ðõð ×,Ñ,Ñ8Ø�J‰JàÜ˜×7Ñ7Ó8ðõð �J‰J˜¤c¨$×*<Ñ*<À×@QÑ@QÑ*QÓ&RÐSÔTÞØ�J‰J˜¬¨_Ó)=Ð>Ô?Ø�
‰
�O¤SÐ):Ó%;Ð<Ô=Ø�;�;˜4Ÿ>Ÿ>ÜÐOÓPÐPØ�[�[Ø�J‰J�zÕ"Ø�^�^Ø�J‰J�}Ô%Ø�
‰
�4×'Ñ'Ô(Ø�
‰
�4×,Ñ,Õ-Ø	˜Ó	˜d×?Ñ?ÔC`ÐabÑCcÓcÜ! -Ñ"?Ó@ˆÔÜ˜,Ó'ˆŒÜ˜T×.Ñ.Ó/ˆŒÜ�4˜¨×/Ñ/Ü" ?Ó3ˆDÔÜ"Ð#4Ó5ˆDÕà$3Ð#4°AÐ6GÐ5HÐ!IˆDÕä! -Ó0ˆÔØÑ(Ü"Ð#4Ó5ˆDÔð # aÑ'×F¬3¨t×/@Ñ/@Ó+AÀQÑ+F€OÞœ>Ð*;×<Ñ<Ø˜1ÓØ"ˆDŒOÜ�MŠMØÐ*Ð+ð ,Dð Dõô "ØEÐFWÐEXð Yð óð ð ‡{‡{�t—~—~ÜÐKÓLÐLØ	��Øˆ�Ø	��ØˆŒä—*‘*—/‘/Ó#€KØ‡z‡zØ/5ˆÐ+Ñ,Ü�d˜I uÓ-€GØ�%Ó˜DŸL™LÑ4Ü×ÑØ.5Ð*Ò+Ü×ÑØ18Ð-Ò.Ü× Ñ Ø29Ð.Ò/Ü× Ñ Ø29Ð.Ò/Ü×ÑØ7>Ð3Ò4Ü×ÑØ4;Ð0Ò1Ü ×"Ñ"Ø5<Ð1Ò2à29Ð.Ñ/ð
Ü'¨×(<Ñ(<×(BÑ(BÓ(DÓEˆô !1°¼r¿w¹w¿¹ÈsÓ?SÓ T€K�ÑÜ03°OÓ0D€KÐ,Ñ-Ø×Ñ×!Ñ!Ó# uÓ,Ü×!Ñ!Üð Eóð ô $ DÓ6ˆÜ14°T×5aÑ5aÓ1b×1hÑ1hÓ1j€KÐ-Ñ.Ø.4€KÐ*Ñ+Ø‡�Ñ"Ü9<¸T¿_¹_Ó9MˆÐ5Ñ6Ø×'Ñ'Ñ3Ü@CÀD×DdÑDdÓ@eˆÐ<Ñ=Ø×ÑÑ)Ü69¸$×:PÑ:PÓ6Q×6WÑ6WÓ6YˆÐ2Ñ3Ø×$Ñ$Ñ0ÜGJÈ4×KhÑKhÓGi×GoÑGoÓGqˆÐCÑDØ× Ñ Ñ,ÜCFÀt×G`ÑG`ÓCa×CgÑCgÓCiˆÐ?Ñ@Ø×ÑÑ'Ü9<¸T×=QÑ=QÓ9R×9XÑ9XÓ9ZˆÐ5Ñ6Ø×"Ñ"Ñ.ÜEHÈ×IdÑIdÓEe×EkÑEkÓEmˆÐAÑBØ×!Ñ!Ñ-Ü:=¸d×>XÑ>XÓ:YˆÐ6Ñ7Ø××Ø14ˆÐ-Ñ.Ø×)Ñ)Ñ5ÜBEÀd×FhÑFhÓBiˆÐ>Ñ?à×"×"Ü;¸DÓNˆàÐÐøôO ó 
ÜØ,¨T×-AÑ-A×-GÑ-GÓ-IÐ,JÐJ[Ô\i×\nÒ\nÓ\pÐ[qÐqrÐsó
ð 	
ð
ús   Ô:#_ ßA`Úpodc                 óú   • U R                   S:X  a*  [        SS9(       a  U R                  (       a  SUS'   OSUS'   U R                  (       a  SUS'   U(       a"  U R                  U l        U R                  U l        X4$ )	zQ
Prepares and returns an environment with the correct TPU environment variables.
Úbf16T)Úcheck_is_tpurN   ÚXLA_DOWNCAST_BF16ÚXLA_USE_BF16rP   rQ   )r‚   r   Údowncast_bf16r~   Útpu_vmÚvmÚtpu_nameÚtpu)r"   r:   rB  s      r)   Úprepare_tpurM  Q  sl   € ð ×Ñ˜vÓ%Ô*@Èd×*SØ××Ø/2ˆKÐ+Ò,à*-ˆK˜Ñ'Ø‡z‡zØ/5ˆÐ+Ñ,Þ
à—+‘+ˆŒØ—=‘=ˆŒØÐÐr+   Únargsc                 óN  • [        U 5      S:  a  0 $ S n[        R                  " 5       nUR                  U 5      u  p4[	        U5       H†  u  pVUR                  S5      (       d  M  S nUS-   [        U5      :  a'  XES-      R                  S5      (       a  [        S5      eO[        S5      eUc  UR                  XaS9  Mw  UR                  XgS9  Mˆ     UR                  U 5      R                  R                  5        VV	s0 s H  u  p‰X‰S;   a  [        U	5      OU	_M     sn	n$ s  sn	nf )	Nr   c                 ój   •  [        U 5      n U S-  U :X  a  [        U 5      $ U $ ! [         a    U s $ f = f)Nr   )ÚfloatrÎ   rp   )Úss    r)   Ú_infer_typeÚ+_convert_nargs_to_dict.<locals>._infer_typej  s=   € ð	Ü�a“ˆAà�A‰v˜‹{Ü˜1“v�ØˆHøÜó 	ØŠHð	ús   ‚# ¡# £2±2)Ú-z--r   ul   SageMaker doesnâ€™t support argparse actions for `store_true` or `store_false`. Please define explicit types)Útype)Úaction)ÚTrueÚFalse)r1   ÚargparseÚArgumentParserr   Ú	enumeraterB   rp   Úadd_argumentÚ
parse_argsÚ__dict__r   r   )
rN  rS  r#   r&   ÚunknownÚindexÚargumentrW  r'   r(   s
             r)   Ú_convert_nargs_to_dictrc  e  s3  € Ü
ˆ5ƒz�Aƒ~Øˆ	òô ×$Ò$Ó&€FØ×(Ñ(¨Ó/�J€AÜ$ WÖ-‰ˆØ×Ñ˜{×+Ó+ØˆFØ�q‰yœ3˜w›<Ó'Ø 1™9Ñ%×0Ñ0°×=Ñ=ä$ð Góð ð >ô !ð Cóð ð ‰~Ø×#Ñ# HÐ#Ó?à×#Ñ# HÐ#Ó<ñ# .ð* !×+Ñ+¨EÓ2×;Ñ;×AÑAÔCôâC‰JˆCð 	Ð.?Ó%?Œl˜5Ô!ÀUÒJÙCòð ùó s   Ã>D!Úsagemaker_configc                 ó"
  • [        S5        U R                  [        R                  S'   U R                  b  U R                  [        R                  S'   O`UR
                  bH  UR                  b;  UR
                  [        R                  S'   UR                  [        R                  S'   O[        S5      e[        R                  R                  UR                  5      nU(       d  Sn[        R                  R                  UR                  5      nUR                  S5      (       d  [        S	U S
35      e[        S5        [        UR                  5      n [!        UR"                  R%                  5       5      n [)        UR*                  R-                  5       5      nS[/        U5      UR0                  UR2                  [/        UR4                  5      [/        UR6                  5      [/        UR8                  5      U R:                  R0                  S.nUR"                  R%                  5       S:X  a%  [=        5       (       d  [?        S5      e[A        X5      nS nU R:                  [B        RD                  :X  a  SSSS000nS n	U RF                  bŒ  [        SU RF                   S35        0 n	[I        U RF                  5       n
[K        U
5       H6  u  p¼US:X  a  M  URM                  S5      nUS   RO                  5       X�S   '   M8     S S S 5        [        SU	 35        S nU RP                  bž  [        SU RP                   S35        / n[I        U RP                  5       n
[K        U
5       HH  u  p¼US:X  a  M  URM                  S5      nUS   US   RO                  5       S.nURS                  U5        MJ     S S S 5        [        SU 35        [        S 5        U RT                  UUU RV                  U RX                  U RZ                  U R\                  U R^                  U R`                  U Rb                  S!UUUUS".nU Rd                  b  [g        U Rd                  U5      nX4$ ! [         a>    [        SUR"                  R%                  5        S[         R&                  " 5        S35      ef = f! [         a>    [        SUR*                  R-                  5        S[(        R&                  " 5        S35      ef = f! , (       d  f       GNó= f! , (       d  f       GNX= f)#Nz(Configuring Amazon SageMaker environmentÚAWS_DEFAULT_REGIONÚAWS_PROFILEÚAWS_ACCESS_KEY_IDÚAWS_SECRET_ACCESS_KEYz]You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profilerb   z.pyz8Your training script should be a python script and not "Ú"z'Converting Arguments to Hyperparametersr`   ra   rf   rP   )ÚACCELERATE_USE_SAGEMAKERrc   rg   rh   ri   rj   rk   Ú%ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPErd   re   ÚsmdistributedÚdataparallelÚenabledTzLoading SageMaker Inputs from z filer   Ú	r   zLoaded SageMaker Inputs: zLoading SageMaker Metrics from )ÚNameÚRegexzLoaded SageMaker Metrics: zCreating EstimatorF)Ú	image_uriÚentry_pointÚ
source_dirÚroleÚtransformers_versionÚpytorch_versionÚ
py_versionÚbase_job_nameÚinstance_countÚinstance_typeÚdebugger_hook_configÚdistributionÚhyperparametersÚenvironmentÚmetric_definitions)4ÚprintÚregionry   rz   ÚprofileÚaws_access_key_idÚaws_secret_access_keyr2   r5  Údirnamerv   ÚbasenameÚendswithrp   rc  rx   r	   r‚   rƒ   r„   r   r†   rE   rD   r(   r‡   rˆ   r‰   rŠ   Údistributed_typer
   r…   rH   r   ÚDATA_PARALLELÚsagemaker_inputs_fileÚopenr\  ÚsplitÚstripÚsagemaker_metrics_filers   rs  Úiam_role_namerw  rx  ry  rz  rq   Úec2_instance_typeÚadditional_argsr   )rd  r"   ru  rt  r  r‚   r†   r€  r~  Úsagemaker_inputsÚfileÚiÚlineÚlÚsagemaker_metricsÚmetric_dicts                   r)   Úprepare_sagemager_args_inputsr›  �  s¯  € ô 
Ð
4Ô5Ø'7×'>Ñ'>„B‡J�JÐ#Ñ$ð ×ÑÑ+Ø$4×$<Ñ$<Œ�
‰
�=Ò!Ø	×	Ñ	Ñ	+°×0JÑ0JÑ0VØ*.×*@Ñ*@Œ�
‰
Ð&Ñ'Ø.2×.HÑ.HŒ�
‰
Ð*Ò+äÐuÓvÐvô —‘—‘ ×!5Ñ!5Ó6€JÞØˆ
Ü—'‘'×"Ñ" 4×#7Ñ#7Ó8€KØ×Ñ ×&Ñ&ÜÐSÐT_ÐS`Ð`aÐbÓcÐcä	Ð
3Ô4Ü,¨T×-FÑ-FÓG€Oð
Ü'¨×(<Ñ(<×(BÑ(BÓ(DÓEˆð
Ü& t×':Ñ':×'@Ñ'@Ó'BÓCˆð %+Ü&)¨/Ó&:Ø%3×%9Ñ%9Ø"&×"2Ñ"2Ü+.¨t×/HÑ/HÓ+IÜ),¨T×-DÑ-DÓ)EÜ69¸$×:^Ñ:^Ó6_Ø1A×1RÑ1R×1XÑ1Xñ	€Kð ×Ñ×!Ñ!Ó# uÓ,Ü×!Ñ!Üð Eóð ô $ DÓ6ˆà€LØ×(Ñ(Ô,D×,RÑ,RÓRØ'¨.¸9ÀdÐ:KÐ)LÐMˆð ÐØ×-Ñ-Ñ9ÜÐ.Ð/?×/UÑ/UÐ.VÐV[Ð\Ô]ØÐÜÐ"×8Ñ8Ô9¸TÜ$ Tž?‘�Ø˜“6ÙØ—J‘J˜tÓ$�Ø)*¨1©¯©«Ð  1¡Ó&ñ	 +÷ :ô 	Ð)Ð*:Ð);Ð<Ô=ð ÐØ×.Ñ.Ñ:ÜÐ/Ð0@×0WÑ0WÐ/XÐX]Ð^Ô_ØÐÜÐ"×9Ñ9Ô:¸dÜ$ Tž?‘�Ø˜“6ÙØ—J‘J˜tÓ$�à˜a™DØ˜q™TŸZ™Z›\ñ�ð "×(Ñ(¨Ö5ñ +÷ ;ô 	Ð*Ð+<Ð*=Ð>Ô?ô 
Ð
Ôà%×/Ñ/Ø"Ø Ø ×.Ñ.Ø 0× EÑ EØ+×;Ñ;Ø&×1Ñ1Ø)×7Ñ7Ø*×7Ñ7Ø)×;Ñ;Ø %Ø$Ø*Ø"Ø/ñ€Dð$ ×'Ñ'Ñ3ÜÐ+×;Ñ;¸TÓBˆØÐ!Ð!øôm ó 
ÜØ,¨T×-AÑ-A×-GÑ-GÓ-IÐ,JÐJ[Ô\i×\nÒ\nÓ\pÐ[qÐqrÐsó
ð 	
ð
ûô ó 
ÜØ& t×':Ñ':×'@Ñ'@Ó'BÐ&CÐCTÔUb×UgÒUgÓUiÐTjÐjkÐló
ð 	
ð
ú÷@ :Ö9ú÷ ;Ö:ús4   Å#Q Å9#R" Ê;AS-ÍAS?ÑARÒ"AS*Ó-
S<Ó?
Tc                 ó  • [         R                  R                  U S5      R                  S5       Vs/ s H  n[	        U5      S:”  d  M  UPM     nnUR                  [        U5      5        SR                  U5      $ s  snf )z‘
Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the
caller to set it in os.environ.
r0   r™   r   )ry   rz   ÚgetrŽ  r1   rs   rD   Újoin)Úenv_var_nameÚpath_to_addÚpÚpathss       r)   r4  r4    sd   € ô
 Ÿ
™
Ÿ™ |°RÓ8×>Ñ>¸sÔCÓRÒC�1ÄsÈ1ÃvÐPQÁz�QÑC€EÐRØ	‡L�L”�[Ó!Ô"Ø�8‰8�E‹?Ðùò Ss   ²A<Á	A<c                   ó(   • \ rS rSrSrSS jrS rSrg)ÚPrepareForLaunchi  aE  
Prepare a function that will launched in a distributed setup.

Args:
    launcher (`Callable`):
        The function to launch.
    distributed_type ([`~state.DistributedType`]):
        The distributed type to prepare for.
    debug (`bool`, *optional*, defaults to `False`):
        Whether or not this is a debug launch.
c                 ó<   • Xl         [        U5      U l        X0l        g )N)Úlauncherr   rŠ  r~   )Úselfr¦  rŠ  r~   s       r)   Ú__init__ÚPrepareForLaunch.__init__  s   € Ø ŒÜ /Ð0@Ó AˆÔØ�
r+   c                 ó˜  • U R                   (       a†  [        [        R                  R	                  S5      5      n[        R                  R	                  S5      n[
        R                  R                  SU[
        R                  R                  XC5      US9  GOU R                  [        R                  [        R                  [        R                  [        R                  [        R                  [        R                   [        R"                  4;   a�  [%        U5      [        R                  S'   [        [        R                  R	                  SS5      5      n[        [        R                  R	                  SS	5      5      n[%        XV-  U-   5      [        R                  S
'   [%        S5      [        R                  S'   U R&                  " U6   g )NÚ
WORLD_SIZEÚACCELERATE_DEBUG_RDV_FILEÚgloo)ÚrankÚstoreÚ
world_sizeÚ
LOCAL_RANKÚNPROCr   Ú	NODE_RANKr   ÚRANKÚFORK_LAUNCHED)r~   rÎ   ry   rz   r�  ÚtorchÚdistributedÚinit_process_groupÚ	FileStorerŠ  r   Ú	MULTI_GPUÚ	MULTI_MLUÚ
MULTI_MUSAÚ	MULTI_NPUÚ	MULTI_XPUÚ	MULTI_CPUÚMULTI_NEURONrD   r¦  )r§  ra  r"   r°  Úrdv_fileÚnprocrÐ   s          r)   Ú__call__ÚPrepareForLaunch.__call__!  s<  € Ø�:�:ÜœRŸZ™ZŸ^™^¨LÓ9Ó:ˆJÜ—z‘z—~‘~Ð&AÓBˆHÜ×Ñ×0Ñ0ØØÜ×'Ñ'×1Ñ1°(ÓGØ%ð	 1ó ð ×"Ñ"Ü×%Ñ%Ü×%Ñ%Ü×&Ñ&Ü×%Ñ%Ü×%Ñ%Ü×%Ñ%Ü×(Ñ(ð'
ó 
ô (+¨5£zŒB�J‰J�|Ñ$ÜœŸ
™
Ÿ™ w°Ó2Ó3ˆEÜœBŸJ™JŸN™N¨;¸Ó:Ó;ˆIÜ!$ UÑ%6¸Ñ%>Ó!?ŒB�J‰J�vÑä&)¨!£fŒ�
‰
�?Ñ#Ø�Š�tÒr+   )r~   rŠ  r¦  N)ÚNOF)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r¨  rÃ  Ú__static_attributes__© r+   r)   r¤  r¤    s   † ñ
ôõ
r+   r¤  )F)9rZ  ry   r3   rt   rÕ   Úastr   Úshutilr   Útypingr   r¶  Úcommands.config.config_argsr   Úutilsr   r	   r
   r   r   r   r   r   r   r   r   Úutils.constantsr   Úutils.otherr   r   r   Úutils.versionsr   r0   r   Údataclassesr   r   r*   r9   Ú	NamespaceÚdictrD   rH   Útupler„   r•   r  r  rA  ÚboolrM  rc  r›  r4  r¤  rÌ  r+   r)   Ú<module>rÚ     s
  ðó Û 	Û Û 
Û Ý Ý Ý ã å 9÷÷ ÷ ñ õ <ß DÑ DÝ -Ý !ß Bð -/ô ò/ð0˜×*Ñ*ð ¸¸cÀ3¸h¹ô ð$b¨(×*<Ñ*<ð bÀÀtÈCÁyÐRVÐWZÐ\_ÐW_ÑR`ÐG`ÑAaô bðJF × 2Ñ 2ð F°t¸CÀ¸H±~ô FðRØ
×
Ñ
ðØ+/ðà
ˆ4�‰9�d˜3 ˜8‘nÐ$Ñ%ôð6a H×$6Ñ$6ð a¸5ÀÀcÁÈDÐQTÐVYÐQYÉNÐAZÑ;[ô aðJ HMñØ
×
Ñ
ðØ+/°°S°©>ðØ@Dðà
ˆ8×Ñ˜t C¨ H™~Ð-Ñ.õð(' $ s¡)ð '°°S¸#°X±ô 'ðTs"Ø%ðs"Ø-5×-?Ñ-?ðs"à
ˆ8×Ñ˜t C¨ H™~Ð-Ñ.ôs"òl÷,ò ,r+   