ó
    EñiÅ  ã                   ó8  • S SK r S SKrS SKrS SKJr  S SKJr  S SKrS SKJ	s  J
r  S SKJrJrJr  SSKJr  SSKJr  SS	KJr  SS
KJrJr  \" 5       (       a®  S SKJrJr  S SKJr  S SKJr  S SK J!r!  S SK"J#r#  S SK$J%r%  S SK&J'r'J(r(  S SK)J*r*  S SK+J,r,  S SK-J.r.  S SK/J0r0J1r1  S SK2J3r3  S SK4J5r5J6r6J7r7J8r8  S SK9J:r:J;r;J<r<J=r=J>r>  S SK?J@r@JArAJBrB  S SKCJDrD  S SKEJFrF  S SKGJHrHJIrIJJrJJKrKJLrLJMrM  S SKNJOrO  S SKPJQrQJRrRJSrSJTrTJUrUJVrV  S SKWJXrXJYrYJZrZ  S<S  jr[S! r\ " S" S#5      r]S$ r^S% r_ " S& S'\5      r`S( ra " S) S*5      rb " S+ S,\5      rcS- rd " S. S/\5      re " S0 S1\e5      rf " S2 S3\e5      rg " S4 S5\e5      rhS6 riS=S7 jrj " S8 S9\R                  RÖ                  5      rlS: rmS; rng)>é    N)ÚABC)Úpartial)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )ÚAcceleratedOptimizer)ÚAcceleratedScheduleré   )Úis_megatron_lm_available)Úrecursively_applyÚsend_to_device)ÚmpuÚtensor_parallel)ÚDistributedDataParallel)Úfinalize_model_grads)Ú	ModelType)Úget_num_microbatches)Úget_megatron_optimizer)Úget_tensor_model_parallel_groupÚ"get_tensor_model_parallel_src_rank)Úget_forward_backward_func)Úget_model_config)Úbuild_train_valid_test_datasets)Ú	BertModelÚT5Model)ÚClassification)Úget_argsÚget_tensorboard_writerÚget_tokenizerÚprint_rank_last)Ú_add_data_argsÚ_add_validation_argsÚ!core_transformer_config_from_argsÚ
parse_argsÚvalidate_args)Úload_args_from_checkpointÚload_checkpointÚsave_checkpoint)Úset_global_variables)Úgpt_builder)Ú_compile_dependenciesÚ_init_autoresumeÚ_initialize_distributedÚ_set_random_seedÚset_jit_fusion_optionsÚwrite_args_to_tensorboard)Ú_vocab_size_with_padding)Ú%build_train_valid_test_data_iteratorsÚget_optimizer_param_schedulerÚnum_floating_point_operationsÚsetup_model_and_optimizerÚ
train_stepÚtraining_log)Ú)average_losses_across_data_parallel_groupÚcalc_params_l2_normÚget_ltor_masks_and_position_idsc           
      óT  • [        5       nUR                  (       a  SOSnUR                  S:X  a'  [        SUR                   SU S35        [        S5        [        U5      nUR                  S:X  aZ  UR                  (       a/  UR                  (       a  S	OSn[        UUUR                  S
U US9nU$ [        UUR                  S	U US9n U$ UR                  S:X  a  SUl
        [        X@USSS9nU$ UR                  S:X  a  [        USS
U UUUS9nU$ [        SUR                   35      e)zBuild the model.zpre-trainingzfine-tuningr   z	Building z model in the z mode.z»The Megatron LM model weights are initialized at random in `accelerator.prepare`. Please use `accelerator.load_checkpoint` to load a pre-trained checkpoint matching the distributed setup.Úbertr   T)ÚconfigÚnum_tokentypesÚadd_binary_headÚparallel_outputÚpre_processÚpost_process)r>   Únum_classesr?   rB   rC   ÚgptFN)Úvp_stager>   Út5)r>   r?   rA   rB   rC   Úadd_encoderÚadd_decoderúUnsupported model type: )r   Úpretraining_flagÚrankÚprintÚmodel_type_namer$   Úbert_binary_headr   r   Ú
num_labelsÚuse_legacy_modelsr+   r   Ú
ValueError)	rB   rC   rH   rI   ÚargsÚmoder>   r?   Úmodels	            ÚY/home/mande/repo/quber/.venv/lib/python3.13/site-packages/accelerate/utils/megatron_lm.pyÚmodel_provider_funcrW   U   sS  € ä‹:€DØ!×2×2‰>¸€DØ‡y�y�Aƒ~Ü�	˜$×.Ñ.Ð/¨~¸d¸VÀ6ÐJÔKÜðxô	
ô /¨tÓ4€FØ×Ñ˜vÓ%Ø× × Ø"&×"7×"7™Q¸QˆNÜØØ-Ø $× 5Ñ 5Ø $Ø'Ø)ñˆEð@ €Lô/ #ØØ ŸO™OØ Ø'Ø)ñ‰Eð. €Lð! 
×	Ñ	 Ó	&à!&ˆÔÜ˜D¨|ÀdÐSWÑXˆð €Lð 
×	Ñ	 Ó	%ÜØØØ Ø#Ø%Ø#Ø#ñ
ˆð €Lô Ð3°D×4HÑ4HÐ3IÐJÓKÐKó    c                 ó¼  • U R                  S5        [        5       nU R                  R                  R                  b‡  U R                  R                  R
                  c  [        S5      eU R                  R                  R
                  nU R                  R                  R	                  U5      n[        X5      n[        XS S9nO†[        R                  nUR                  S:X  a  [        R                  n[        nU R                  R                  R
                  b   U R                  R                  R
                  n[        UU5      u  p4n[        U5      Ul        X4U4$ )Nz#Preparing model optimizer schedulerzaYou must provide a `custom_model_provider_function` when using a `custom_prepare_model_function`.)Ú	schedulerrG   )rM   r   ÚstateÚmegatron_lm_pluginÚcustom_prepare_model_functionÚcustom_model_provider_functionrR   Úprepare_optimizerÚprepare_schedulerr   Úencoder_or_decoderrN   Úencoder_and_decoderrW   r6   ÚlenÚ	model_len)ÚacceleratorrS   Úcustom_model_provider_funcrU   Ú	optimizerrZ   Ú
model_typeÚmodel_provider_func_s           rV   Ú!prepare_model_optimizer_schedulerrj   †   s,  € Ø×ÑÐ;Ô<Ü‹:€DØ×Ñ×+Ñ+×IÑIÑUØ×Ñ×/Ñ/×NÑNÑVÜØsóð ð &1×%6Ñ%6×%IÑ%I×%hÑ%hÐ"Ø×!Ñ!×4Ñ4×RÑRÐSmÓnˆÜ% kÓ9ˆ	Ü% kÈÑM‰	ä×1Ñ1ˆ
Ø×Ñ 4Ó'Ü"×6Ñ6ˆJÜ2ÐØ×Ñ×/Ñ/×NÑNÑZØ#.×#4Ñ#4×#GÑ#G×#fÑ#fÐ Ü(AØ Øó)
Ñ%ˆ˜9ô ˜“Z€D„NØ˜YÐ&Ð&rX   c                   ó0   • \ rS rSrSrS rS rS rS rSr	g)	ÚMegatronLMDummyDataLoaderé¢   zª
Dummy dataloader presents model parameters or param groups, this is primarily used to follow conventional training

Args:
    **dataset_kwargs: Megatron data arguments.
c                 óô   • [         R                  " 5       n[        U5      n[        U5      nUR	                  5       n[        US   5      U l        U R                  R                  U5        SU R                  S'   g )Nr   TÚmegatron_dataset_flag)ÚargparseÚArgumentParserr"   r#   Úparse_known_argsÚvarsÚdataset_argsÚupdate)ÚselfÚdataset_kwargsÚparserÚ	data_argss       rV   Ú__init__Ú"MegatronLMDummyDataLoader.__init__ª   sh   € Ü×(Ò(Ó*ˆÜ Ó'ˆÜ% fÓ-ˆØ×+Ñ+Ó-ˆ	Ü  ¨1¡Ó.ˆÔØ×Ñ× Ñ  Ô0Ø59ˆ×ÑÐ1Ò2rX   c                 óÆ   • [        5       nU R                  R                  5        H9  u  p#[        XS5      nXC:w  a  [	        SU SU SU SU 35        [        XU5        M;     g )NÚ z<WARNING: MegatronLMDummyDataLoader overriding arguments for Ú:ú with )r   rt   ÚitemsÚgetattrrM   Úsetattr)rv   rS   ÚkeyÚvalueÚ	old_values        rV   Úset_megatron_data_argsÚ0MegatronLMDummyDataLoader.set_megatron_data_args³   sq   € Ü‹zˆØ×+Ñ+×1Ñ1Ö3‰JˆCÜ ¨2Ó.ˆIØÓ!ÜØRÐSVÐRWÐWXÐYbÐXcÐciÐjmÐinÐnoÐpuÐovÐwôô �D˜uÖ%ò 4rX   c                 ó‚  • S nUR                   R                  R                  b   UR                   R                  R                  $  [        5       nUR                  S:X  a  SSKJn  SUl        U$ UR                  S:X  a  SSKJn  SUl        U$ UR                  S:X  a  SSK	Jn  SUl        U$  U$ ! [         a     U$ f = f)Nc                 óZ  • [        5       n[        UR                  [        [        45      (       a  UR                  OUR                  /UR
                  U UR                  S.nUR                  S:X  a)  UR                  UR                  UR                  S.5        O€UR                  S:X  a  UR                  SUR                  05        ORUR                  S:X  a*  UR                  UR                  UR                  SS.5        O[        SUR                   35      e[        S	0 UD6u  p4nX4U4$ )
z&Build train, valid, and test datasets.)Údata_prefixÚsplits_stringÚtrain_valid_test_num_samplesÚseedr=   )Úmax_seq_lengthÚbinary_headrE   rŽ   rG   )rŽ   Úmax_seq_length_decÚdataset_typerJ   © )r   Ú
isinstanceÚ	data_pathÚlistÚtupleÚsplitr�   rN   ru   Ú
seq_lengthrO   Úencoder_seq_lengthÚdecoder_seq_lengthrR   r   )Útrain_val_test_num_samplesrS   rt   Útrain_dsÚvalid_dsÚtest_dss         rV   Ú"train_valid_test_datasets_providerÚlMegatronLMDummyDataLoader.get_train_valid_test_datasets_provider.<locals>.train_valid_test_datasets_provider¾   s  € ä“:ˆDä1;¸D¿N¹NÌTÔSXÈM×1ZÑ1Z˜tŸ~š~Ðae×aoÑaoÐ`pØ!%§¡Ø0JØŸ	™	ñ	ˆLð ×#Ñ# vÓ-Ø×#Ñ#à*.¯/©/Ø'+×'<Ñ'<ñõð ×%Ñ%¨Ó.Ø×#Ñ#à(¨$¯/©/ðõð
 ×%Ñ%¨Ó-Ø×#Ñ#à*.×*AÑ*AØ.2×.EÑ.EØ(,ñõô !Ð#;¸D×<PÑ<PÐ;QÐ!RÓSÐSÜ*IÑ*YÈLÑ*YÑ'ˆH Ø wÐ.Ð.rX   r=   r   )rŸ   TrE   rG   )r[   r\   Ú*custom_megatron_datasets_provider_functionr   rN   Úpretrain_bertrŸ   Úis_distributedÚpretrain_gptÚpretrain_t5ÚImportError)rv   re   rŸ   rS   s       rV   Ú&get_train_valid_test_datasets_providerÚ@MegatronLMDummyDataLoader.get_train_valid_test_datasets_provider½   sÍ   € ò!	/ðF ×Ñ×/Ñ/×ZÑZÑfØ×$Ñ$×7Ñ7×bÑbÐbð	Ü“:ˆDà×#Ñ# vÓ-ÝLàDHÐ2ÔAØ9Ð9Ø×%Ñ%¨Ó.ÝKàDHÐ2ÔAØ9Ð9Ø×%Ñ%¨Ó-ÝJàDHÐ2ÔAØ9Ð9ð	 .ð 2Ð1øô ó 	ØØ1Ð1ð	ús   Á(B0 Á/B0 ÂB0 Â0
B>Â=B>c                 óx  • [        5       nU R                  U5      nUR                  b�  / n/ n/ n[        [	        USS5      5       H`  n[
        R                  " U5        [        U5      nUR                  US   5        UR                  US   5        UR                  US   5        Mb     O[        U5      u  pEnXEU4$ )Nrd   r   r   r   )	r   r§   Ú$virtual_pipeline_model_parallel_sizeÚranger�   r   Ú(set_virtual_pipeline_model_parallel_rankr3   Úappend)	rv   re   rS   Ú!train_valid_test_dataset_providerÚtrain_data_iteratorÚvalid_data_iteratorÚtest_data_iteratorÚiÚ	iteratorss	            rV   r3   Ú?MegatronLMDummyDataLoader.build_train_valid_test_data_iteratorsù   sÇ   € Ü‹zˆà,0×,WÑ,WÐXcÓ,dÐ)Ø×4Ñ4Ñ@Ø"$ÐØ"$ÐØ!#ÐÜœ7 4¨°aÓ8Ö9�Ü×<Ò<¸QÔ?ÜAÐBcÓd�	Ø#×*Ñ*¨9°Q©<Ô8Ø#×*Ñ*¨9°Q©<Ô8Ø"×)Ñ)¨)°A©,Ö7ò :ô LqØ1óLÑHÐÐ6Hð #Ð9KÐKÐKrX   )rt   N)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rz   r†   r§   r3   Ú__static_attributes__r’   rX   rV   rl   rl   ¢   s   † ñò:ò&ò:2õxLrX   rl   c                 ó  •  " S S5      nUS L n[         R                  " U[         R                  U R                  S9n[         R                  R                  U[        5       [        5       S9  U(       d  U(       a  U" 5       $ U$ )Nc                   ó    • \ rS rSrS rS rSrg)Ú?_handle_megatron_data_iterator.<locals>.DummyMegatronDataloaderi  c                 ó   • U $ ©Nr’   ©rv   s    rV   Ú__iter__ÚH_handle_megatron_data_iterator.<locals>.DummyMegatronDataloader.__iter__  s   € ØˆKrX   c                 ó   • 0 $ r¿   r’   rÀ   s    rV   Ú__next__ÚH_handle_megatron_data_iterator.<locals>.DummyMegatronDataloader.__next__  s   € ØˆIrX   r’   N)rµ   r¶   r·   r¸   rÁ   rÄ   rº   r’   rX   rV   ÚDummyMegatronDataloaderr½     s   † ò	õ	rX   rÆ   ©ÚdtypeÚdevice©Úgroup)ÚtorchÚtensorÚboolrÉ   ÚdistributedÚ	broadcastr   r   )re   Údata_iteratorrÆ   Úis_data_iterator_emptyÚis_src_data_iterator_emptys        rV   Ú_handle_megatron_data_iteratorrÔ     sw   € ÷ñ ð +¨dÐ2ÐÜ!&§¢Ð.DÌEÏJÉJÐ_j×_qÑ_qÑ!rÐÜ	×Ñ×ÑØ"Ô$FÓ$HÔPoÓPqð  ñ ö &Ö*@Ù&Ó(Ð(ØÐrX   c                 ó0  • U R                  S5        [        5       nUR                  (       Gd3  SSKJnJn  UR                  UR                  -  nU Vs0 s H  of[        XX6   5      _M     nnUS   cS  [        US   [        R                  R                  R                  5      (       a
  XWS   l        OUS	 US	 US	 XWS   l        OUS	 XWS'   [        R                  R                  R                  " UR                   40 UD6nU" UU R"                  [$        R&                  " 5       [$        R(                  " 5       SS	U R*                  R-                  5       U R.                  S
9$ UR0                  b   UR0                  u  Ul        Ul        Ul        OSu  Ul        Ul        Ul        UR                  UR                  -  Ul        UR9                  U 5      u  nn	n
UR                  UR                  -  Ul        [;        XS9n[;        X	S9n	[;        X
S9n
X‰U
4$ s  snf )NzPreparing dataloaderr   )Ú_PYTORCH_DATALOADER_KWARGSÚprepare_data_loaderÚ
batch_sizeÚsamplerÚshuffleÚbatch_samplerFT)Únum_processesÚprocess_indexÚsplit_batchesÚput_on_deviceÚ	rng_typesÚdispatch_batches)r   r   r   )re   rÑ   )rM   r   ro   Údata_loaderrÖ   r×   Úmicro_batch_sizeÚnum_micro_batchesr�   r“   rÌ   ÚutilsÚdataÚBatchSamplerrØ   Ú
DataLoaderÚdatasetrÉ   r   Úget_data_parallel_world_sizeÚget_data_parallel_rankrà   Úcopyrá   Úconsumed_samplesÚconsumed_train_samplesÚconsumed_valid_samplesÚconsumed_test_samplesr3   rÔ   )re   Ú
dataloaderrS   rÖ   r×   rã   ÚkÚkwargsr¯   r°   r±   s              rV   r×   r×   !  s  € Ø×ÑÐ,Ô-Ü‹:€DØ×%×%Ð%ßQà×0Ñ0°4×3IÑ3IÑIÐÙTnÓoÒTnÈq”W˜ZÐ,FÑ,IÓJÒJÑTnˆÐoØ�,ÑÑ'Ü˜& Ñ+¬U¯[©[×-=Ñ-=×-JÑ-J×KÑKØ/?�yÑ!Õ,à˜9Ð%Ø˜9Ð%Ø˜<Ð(Ø5E�Ñ'Õ2à�Ð'Ø#3�<Ñ ä—[‘[×%Ñ%×0Ò0°×1CÑ1CÑNÀvÑNˆ
ñ #ØØ×ÑÜ×:Ò:Ó<Ü×4Ò4Ó6ØØØ!×+Ñ+×0Ñ0Ó2Ø(×9Ñ9ñ	
ð 		
ð × Ñ Ñ,ð
 ×%Ñ%ñ	ØÔ+ØÔ+ØÕ*ð dkÑ`ˆDÔ'¨Ô)DÀdÔF`Ø $× 5Ñ 5¸×8NÑ8NÑ NˆÔð ×<Ñ<¸[ÓIñ		
ØØØà $× 5Ñ 5¸×9OÑ9OÑ OˆÔä<Ø#ñ
Ðô =Ø#ñ
Ðô <ÈÑvÐà"Ð9KÐKÐKùòo ps   ÁHc                   óH   ^ • \ rS rSrU 4S jrSS jrS r\S 5       rSr	U =r
$ )ÚMegatronLMOptimizerWrapperic  c                 ó$   >• [         TU ]  USS S9  g )NF)Údevice_placementÚscaler©Úsuperrz   )rv   rg   Ú	__class__s     €rV   rz   Ú#MegatronLMOptimizerWrapper.__init__d  s   ø€ Ü‰Ñ˜°UÀ4ÐÒHrX   c                 ó   • g r¿   r’   )rv   Úset_to_nones     rV   Ú	zero_gradÚ$MegatronLMOptimizerWrapper.zero_gradg  ó   € ØrX   c                 ó   • g r¿   r’   rÀ   s    rV   ÚstepÚMegatronLMOptimizerWrapper.stepj  r  rX   c                 ó.   • U R                   R                  $ )zTWhether or not the optimizer step was done, or skipped because of gradient overflow.)rg   Úskipped_iterrÀ   s    rV   Ústep_was_skippedÚ+MegatronLMOptimizerWrapper.step_was_skippedm  s   € ð �~‰~×*Ñ*Ð*rX   r’   r¿   )rµ   r¶   r·   r¸   rz   rÿ   r  Úpropertyr  rº   Ú__classcell__©rû   s   @rV   rõ   rõ   c  s'   ø† õIôòð ñ+ó ö+rX   rõ   c                 óŽ   • U R                  S5        [        5       n[        XR                  UR                  UR
                  5      $ )NzPreparing optimizer)rM   r   r   Úno_wd_decay_condÚscale_lr_condÚlr_mult)re   rU   rS   s      rV   r_   r_   s  s:   € Ø×ÑÐ+Ô,Ü‹:€DÜ! %×)>Ñ)>À×@RÑ@RÐTX×T`ÑT`ÓaÐarX   c                   ó"   • \ rS rSrSrSS jrSrg)ÚMegatronLMDummyScheduleriz  aÞ  
Dummy scheduler presents model parameters or param groups, this is primarily used to follow conventional training
loop when scheduler config is specified in the deepspeed config file.

Args:
    optimizer (`torch.optim.optimizer.Optimizer`):
        The optimizer to wrap.
    total_num_steps (int):
        Total number of steps.
    warmup_num_steps (int):
        Number of steps for warmup.
    **kwargs (additional keyword arguments, *optional*):
        Other arguments.
Nc                 ó4   • Xl         X l        X0l        X@l        g r¿   )rg   Útotal_num_stepsÚwarmup_num_stepsró   )rv   rg   r  r  ró   s        rV   rz   Ú!MegatronLMDummyScheduler.__init__Š  s   € Ø"ŒØ.ÔØ 0ÔØ�rX   )ró   rg   r  r  ©Nr   )rµ   r¶   r·   r¸   r¹   rz   rº   r’   rX   rV   r  r  z  s   † ñ÷rX   r  c                   ó.   ^ • \ rS rSrU 4S jrS rSrU =r$ )ÚMegatronLMSchedulerWrapperi‘  c                 ó$   >• [         TU ]  X5        g r¿   rù   )rv   rZ   Ú
optimizersrû   s      €rV   rz   Ú#MegatronLMSchedulerWrapper.__init__’  s   ø€ Ü‰Ñ˜Õ/rX   c                 ó   • g r¿   r’   )rv   rS   ró   s      rV   r  ÚMegatronLMSchedulerWrapper.step•  s   € ØrX   r’   )rµ   r¶   r·   r¸   rz   r  rº   r
  r  s   @rV   r  r  ‘  s   ø† õ0÷ð rX   r  c                 ó>   • U R                  S5        [        U5      nU$ )NzPreparing scheduler)rM   r4   )re   rg   rZ   s      rV   r`   r`   ™  s!   € Ø×ÑÐ+Ô,Ü-¨iÓ8€IØÐrX   c                   ó>   ^ • \ rS rSrSrU 4S jrS rS rS rSr	U =r
$ )ÚAbstractTrainStepiŸ  z;Abstract class for batching, forward pass and loss handler.c                 ó.   >• [         TU ]  5         Xl        g r¿   )rú   rz   Úname)rv   r"  rû   s     €rV   rz   ÚAbstractTrainStep.__init__¢  s   ø€ Ü‰ÑÔØ�	rX   c                 ó   • g r¿   r’   )rv   re   ro   s      rV   Úget_batch_funcÚ AbstractTrainStep.get_batch_func¦  r  rX   c                 ó   • g r¿   r’   rÀ   s    rV   Úget_forward_step_funcÚ'AbstractTrainStep.get_forward_step_func©  r  rX   c                 ó   • g r¿   r’   )rv   re   s     rV   Úget_loss_funcÚAbstractTrainStep.get_loss_func¬  r  rX   )r"  )rµ   r¶   r·   r¸   r¹   rz   r%  r(  r+  rº   r
  r  s   @rV   r   r   Ÿ  s   ø† ÙEõòò÷ð rX   r   c                   ó>   ^ • \ rS rSrSrU 4S jrS rS rS rSr	U =r
$ )ÚBertTrainStepi°  zW
Bert train step class.

Args:
    args (`argparse.Namespace`): Megatron-LM arguments.
c                 óZ  >• [         TU ]  S5        U R                  XR                  5      U l        U R                  XR                  UR                  5      U l        U R                  UR                  UR                  5      U l        UR                  (       d  S U l        g SSKJn  X0l        g )Nr.  r   )ÚSequenceClassifierOutput)rú   rz   r%  ro   Ú	get_batchr+  rK   rP   Ú	loss_funcr(  rO   Úforward_stepÚmodel_return_dictÚmodel_output_classÚtransformers.modeling_outputsr0  )rv   re   rS   r0  rû   s       €rV   rz   ÚBertTrainStep.__init__¸  s„   ø€ Ü‰Ñ˜Ô)Ø×,Ñ,¨[×:TÑ:TÓUˆŒØ×+Ñ+¨K×9NÑ9NÐPT×P_ÑP_Ó`ˆŒØ ×6Ñ6°t×7LÑ7LÈd×NcÑNcÓdˆÔØ×%×%Ø&*ˆDÕ#åNà&>Õ#rX   c                 óÖ   • S nS nUR                   R                  R                  b   UR                   R                  R                  $ U(       a	   SSKJn  U$ U$ ! [
         a     U$ f = f)Nc                 óh  • / SQn[         R                  nU b  [        U 5      nOSn[        R                  " XU5      nUS   R                  5       nUS   R                  5       nUS   R                  5       nUS   R                  5       nUS   R                  5       n	US   R                  5       n
XVXxXš4$ )	úBuild the batch.)ÚtextÚtypesÚlabelsÚ	is_randomÚ	loss_maskÚpadding_maskNr;  r<  r>  r?  r=  r@  ©rÌ   Úint64Únextr   Úbroadcast_dataÚlongÚfloat)rÑ   ÚkeysÚdatatyperæ   Údata_bÚtokensr<  Úsentence_orderr?  Ú	lm_labelsr@  s              rV   Úget_batch_megatronÚ8BertTrainStep.get_batch_func.<locals>.get_batch_megatronÅ  s»   € ò YˆDÜ—{‘{ˆHð Ñ(Ü˜MÓ*‘à�Ü$×3Ò3°DÀÓIˆFð ˜F‘^×(Ñ(Ó*ˆFØ˜7‘O×(Ñ(Ó*ˆEØ# KÑ0×5Ñ5Ó7ˆNØ˜{Ñ+×1Ñ1Ó3ˆIØ˜xÑ(×-Ñ-Ó/ˆIØ! .Ñ1×6Ñ6Ó8ˆLà .¸YÐTÐTrX   c                 ó´  • [        U 5      n[        U[        R                  R	                  5       5      nUS   R                  5       nUS   R                  5       nSU;   a  US   R                  5       nOSnSU;   a9  US   R                  5       nUS   S:g  R                  [        R                  5      nOSnSnSU;   a  US   R                  5       nOSnX$XvXS4$ )r:  Ú	input_idsÚattention_maskÚtoken_type_idsNr=  éœÿÿÿÚnext_sentence_label)rC  r   rÌ   ÚcudaÚcurrent_devicerE  ÚtorF  )rÑ   ræ   rJ  r@  r<  rL  r?  rK  s           rV   Úget_batch_transformerÚ;BertTrainStep.get_batch_func.<locals>.get_batch_transformerÝ  sÞ   € ä˜Ó&ˆDÜ! $¬¯
©
×(AÑ(AÓ(CÓDˆDð ˜+Ñ&×+Ñ+Ó-ˆFØÐ 0Ñ1×6Ñ6Ó8ˆLØ 4Ó'ØÐ-Ñ.×3Ñ3Ó5‘à�Ø˜4ÓØ  ™N×/Ñ/Ó1�	Ø! (™^¨tÑ3×7Ñ7¼¿¹ÓD‘	à �	Ø �	Ø$¨Ó,Ø!%Ð&;Ñ!<×!AÑ!AÓ!C‘à!%�à .¸YÐTÐTrX   r   ©r1  )r[   r\   Úcustom_get_batch_functionr¢   r1  r¦   ©rv   re   ro   rM  rX  r1  s         rV   r%  ÚBertTrainStep.get_batch_funcÄ  ss   € ò	Uò0	Uð2 ×Ñ×/Ñ/×IÑIÑUØ×$Ñ$×7Ñ7×QÑQÐQÞ ðå3à Ð ð
 )Ð(øô	 ó ØØ%Ð%ðúó   ÁA Á
A(Á'A(c                 ó²   ^ ^• S nUU 4S jnUR                   R                  R                  b   UR                   R                  R                  $ U(       a  U$ U$ )Nc                 óì  • Uu  p4UR                  5       nU R                  5       n [        R                  " UR                  S5      U R	                  S5      -  5      U R                  5       -  nUbo  [
        R                  " UR                  SS5      R                  5       UR                  S5      SS9nUR                  5       nXV-   n[        XV/5      nXxS   US   S.4$ Un[        U/5      nUSUS   04$ )Néÿÿÿÿr   )Úignore_indexr   r   )úlm losszsop lossrc  )rF  rÌ   ÚsumÚviewÚreshapeÚFÚcross_entropyr9   )	r?  rK  Úoutput_tensorÚlm_loss_Ú
sop_logitsÚlm_lossÚsop_lossÚlossÚaveraged_lossess	            rV   Úloss_func_pretrainÚ7BertTrainStep.get_loss_func.<locals>.loss_func_pretrain  sï   € Ø#0Ñ ˆHà—~‘~Ó'ˆHØ!Ÿ™Ó)ˆIÜ—i’i §¡¨bÓ 1°I×4EÑ4EÀbÓ4IÑ IÓJÈYÏ]É]Ë_Ñ\ˆGàÑ%ÜŸ?š?¨:¯?©?¸2¸qÓ+A×+GÑ+GÓ+IÈ>×K^ÑK^Ð_aÓKbÐqsÑt�Ø#Ÿ>™>Ó+�ØÑ)�Ü"KÈWÐL_Ó"`�Ø¸Ñ);ÈÐYZÑI[Ñ\Ð\Ð\ð �Ü"KÈWÈIÓ"V�Ø˜i¨¸Ñ);Ð<Ð<Ð<rX   c                 ó¤  >• TS:X  a2  [        5       nU" UR                  S5      U R                  S5      5      nOƒTR                  S:”  aa  U R                  [        R
                  [        R                  4;   a3  [        5       nU" UR                  ST5      U R                  S5      5      nO[        5       nU" X5      n[        U/5      nUSUS   04$ )Nr   ra  rn  r   )
r   re  rP   rÈ   rÌ   rE  Úintr   r   r9   )r=  ÚlogitsÚloss_fctrn  ro  rP   rv   s        €€rV   Úloss_func_finetuneÚ7BertTrainStep.get_loss_func.<locals>.loss_func_finetune  s«   ø€ Ø˜Q‹ä"›9�Ù §¡¨B£°·±¸R³ÓA‘Ø—‘ 1Ó$¨&¯,©,¼5¿:¹:ÄuÇyÁyÐ:QÓ*QÜ+Ó-�Ù §¡¨B°
Ó ;¸V¿[¹[È»_ÓM‘ä,Ó.�Ù Ó/�ÜGÈÈÓOˆOØ˜& /°!Ñ"4Ð5Ð5Ð5rX   ©r[   r\   Úcustom_loss_function)rv   re   rK   rP   rp  rv  s   `  `  rV   r+  ÚBertTrainStep.get_loss_func  sN   ù€ ò	=ö&	6ð ×Ñ×/Ñ/×DÑDÑPØ×$Ñ$×7Ñ7×LÑLÐLÞØ%Ð%à%Ð%rX   c                 ó   ^ ^^• UUU 4S jnU$ )Nc                 óÊ   >• TR                  U 5      u  p#pEpgT
(       d  SnT(       a  U" X'X6S9nU[        TR                  XT5      4$ U" X'US9n	U	[        TR                  U5      4$ )úForward step.N©Útokentype_idsrL  )r  ©r1  r   r2  )rÑ   rU   rJ  r<  rK  r?  r=  r@  ri  rt  rO   rK   rv   s             €€€rV   r3  Ú9BertTrainStep.get_forward_step_func.<locals>.forward_step.  sh   ø€ àMQÏ^É^Ð\iÓMjÑJˆF˜>°fÞ#Ø�æÙ % fÈ%Ñ b�Ø$¤g¨d¯n©n¸iÓ&XÐXÐXá˜vÀ5ÑI�Øœw t§~¡~°vÓ>Ð>Ð>rX   r’   )rv   rK   rO   r3  s   ``` rV   r(  Ú#BertTrainStep.get_forward_step_func-  s   ú€ ÷	?ð ÐrX   ©r3  r1  r2  r5  ©rµ   r¶   r·   r¸   r¹   rz   r%  r+  r(  rº   r
  r  s   @rV   r.  r.  °  s#   ø† ñõ
?ò>)ò@'&÷Rð rX   r.  c                   ó>   ^ • \ rS rSrSrU 4S jrS rS rS rSr	U =r
$ )ÚGPTTrainStepi>  zV
GPT train step class.

Args:
    args (`argparse.Namespace`): Megatron-LM arguments.
c                 óþ  >• [         TU ]  S5        U R                  XR                  5      U l        U R                  U5      U l        U R                  5       U l        UR                  b  [        5       nUR                  U l        UR                  U l        UR                  U l        UR                  U l        UR                   U l        UR"                  U l        UR$                  (       d  S U l        g SSKJn  X@l        g )Nr†  r   )Ú!CausalLMOutputWithCrossAttentions)rú   rz   r%  ro   r1  r+  r2  r(  r3  Ú
vocab_filer    ÚeodÚ	eod_tokenÚeos_token_idÚ	pad_tokenÚreset_position_idsÚreset_attention_maskÚeod_mask_lossr4  r5  r6  rˆ  )rv   re   rS   Ú	tokenizerrˆ  rû   s        €rV   rz   ÚGPTTrainStep.__init__F  sÅ   ø€ Ü‰Ñ˜Ô(Ø×,Ñ,¨[×:TÑ:TÓUˆŒØ×+Ñ+¨KÓ8ˆŒØ ×6Ñ6Ó8ˆÔØ�?‰?Ñ&Ü%›ˆIØ&Ÿ]™]ˆDŒNØ×*Ñ*ˆŒØ×*Ñ*ˆŒØ"&×"9Ñ"9ˆÔØ$(×$=Ñ$=ˆÔ!Ø!×/Ñ/ˆÔØ×%×%Ø&*ˆDÕ#åWà&GÕ#rX   c                 óä   ^ • U 4S jnU 4S jnUR                   R                  R                  b   UR                   R                  R                  $ U(       a	   SSKJn  U$ U$ ! [
         a     U$ f = f)Nc           
      ó–  >• S/n[         R                  nU b  [        U 5      nOSn[        R                  " XU5      nUS   R                  5       nUSS2SS24   R                  5       nUSS2SS24   R                  5       n[        UTR                  TR                  TR                  TR                  TR                  SS9u  p‰n
XvX˜U
4$ )zGenerate a batchr;  Nr   ra  T©r‹  r�  rŽ  r�  r�  Úpad_mask_loss)rÌ   rB  rC  r   rD  rE  Ú
contiguousr;   r‹  rŽ  r�  r�  )rÑ   rG  rH  ræ   rI  Útokens_r=  rJ  rQ  r?  Úposition_idsrv   s              €rV   rM  Ú7GPTTrainStep.get_batch_func.<locals>.get_batch_megatron[  sÔ   ø€ ð �8ˆDÜ—{‘{ˆHð Ñ(Ü˜MÓ*‘à�Ü$×3Ò3°DÀÓIˆFð ˜V‘n×)Ñ)Ó+ˆGØšQ ¡˜U‘^×.Ñ.Ó0ˆFØšQ   ˜V‘_×/Ñ/Ó1ˆFô 7VØØŸ.™.ØŸ.™.Ø#'×#:Ñ#:Ø%)×%>Ñ%>Ø"×0Ñ0Ø"ñ7Ñ3ˆN |ð  9¸lÐJÐJrX   c           
      óL  >• [        U 5      nSUS   0n[        U[        R                  R	                  5       5      nUS   R                  5       n[        R                  " UR                  S   S4UR                  UR                  S9T	R                  -   n[        R                  " X#/SS9nUS S 2SS 24   R                  5       nUS S 2S S24   R                  5       n[        UT	R                  T	R                  T	R                  T	R                  T	R                   SS9u  pgnXTXvU4$ )	NrP  r   r   rÇ   ©Údimra  Tr•  )rC  r   rÌ   rU  rV  rE  ÚzerosÚshaperÈ   rÉ   r‹  Úconcatr—  r;   rŽ  r�  r�  )
rÑ   ræ   r˜  Úpaddingr=  rJ  rQ  r?  r™  rv   s
            €rV   rX  Ú:GPTTrainStep.get_batch_func.<locals>.get_batch_transformery  s  ø€ Ü˜Ó&ˆDØ  kÑ!2Ð3ˆDÜ! $¬¯
©
×(AÑ(AÓ(CÓDˆDà˜;Ñ'×,Ñ,Ó.ˆGÜ—k’k 7§=¡=°Ñ#3°QÐ"7¸w¿}¹}ÐU\×UcÑUcÑdÐgk×guÑguÑuˆGÜ—l’l GÐ#5¸1Ñ=ˆGØšQ ¡˜U‘^×.Ñ.Ó0ˆFØšQ   ˜V‘_×/Ñ/Ó1ˆFä6UØØŸ.™.ØŸ.™.Ø#'×#:Ñ#:Ø%)×%>Ñ%>Ø"×0Ñ0Ø"ñ7Ñ3ˆN |ð  9¸lÐJÐJrX   r   rZ  )r[   r\   r[  r¤   r1  r¦   r\  s   `     rV   r%  ÚGPTTrainStep.get_batch_funcZ  st   ø€ õ	Kõ<	Kð, ×Ñ×/Ñ/×IÑIÑUØ×$Ñ$×7Ñ7×QÑQÐQÞ ðå2à Ð ð
 )Ð(øô	 ó ØØ%Ð%ðús   ÁA! Á!
A/Á.A/c                 óª   ^• [        5       mU4S jnUR                  R                  R                  b   UR                  R                  R                  $ U$ )Nc                 óò  >• TR                   (       a  Uu  p#OUnUR                  5       nU R                  S5      R                  5       n TR                  S:”  a§  [        R
                  " [        R                  " UR                  S5      U -  5      R                  S5      U R                  5       R                  S5      /5      n[        R                  R                  U[        R                  " 5       S9  US   US   -  nO9[        R                  " UR                  S5      U -  5      U R                  5       -  nTR                  (       au  [        R                  R                  5       nUR                  5       (       aB   SU S[        R                  R                  5        S[         R"                  " 5       S    35       e[%        U/5      nSUS   0nTR                   (       a  UR'                  S	W05        XG4$ )
Nra  r   rÊ   r   zRank z7: found NaN in local forward loss calculation. Device: z, node: rc  rt  )Úreturn_logitsrF  re  Úcontext_parallel_sizerÌ   Úcatrd  rÏ   Ú
all_reducer   Úget_context_parallel_groupÚcheck_for_nan_in_loss_and_gradÚget_rankÚisnanrU  rV  ÚosÚunamer9   ru   )	r?  ri  Úlossesrt  rn  Úglobal_rankÚaveraged_lossÚoutput_dictrS   s	           €rV   r2  Ú-GPTTrainStep.get_loss_func.<locals>.loss_func   s�  ø€ Ø×!×!Ø!.‘�˜à&�Ø—\‘\“^ˆFØ!Ÿ™ rÓ*×0Ñ0Ó2ˆIØ×)Ñ)¨AÓ-Ü—y’y¤%§)¢)¨F¯K©K¸«O¸iÑ,GÓ"H×"MÑ"MÈaÓ"PÐR[×R_ÑR_ÓRa×RfÑRfÐghÓRiÐ!jÓk�Ü×!Ñ!×,Ñ,¨T¼×9WÒ9WÓ9YÐ,ÑZØ˜A‘w  a¡Ñ(‘ä—y’y §¡¨R£°9Ñ!<Ó=À	ÇÁÃÑO�ð ×2×2Ü#×/Ñ/×8Ñ8Ó:�ØŸ:™:Ÿ<™<ð Ø˜K˜=ð )Ü$Ÿz™z×8Ñ8Ó:Ð;¸8ÄBÇHÂHÃJÈqÁMÀ?ðTóÐ'ô FÀtÀfÓMˆMà$ m°AÑ&6Ð7ˆKØ×!×!Ø×"Ñ" H¨fÐ#5Ô6ØÐ$Ð$rX   )r   r[   r\   ry  )rv   re   r2  rS   s      @rV   r+  ÚGPTTrainStep.get_loss_func�  sG   ø€ Ü‹zˆõ	%ð< ×Ñ×/Ñ/×DÑDÑPØ×$Ñ$×7Ñ7×LÑLÐLØÐrX   c                 ó   ^ • U 4S jnU$ )Nc                 ól   >• TR                  U 5      u  p#pEnU" X&XSS9nU[        TR                  U5      4$ )r}  )r=  r€  )	rÑ   rU   rJ  r=  r?  rQ  r™  ri  rv   s	           €rV   r3  Ú8GPTTrainStep.get_forward_step_func.<locals>.forward_stepÃ  s?   ø€ ð GKÇnÁnÐUbÓFcÑCˆF˜I°|Ù! &¸ÑVˆMà ¤'¨$¯.©.¸)Ó"DÐDÐDrX   r’   ©rv   r3  s   ` rV   r(  Ú"GPTTrainStep.get_forward_step_funcÂ  s   ø€ õ	Eð ÐrX   )	r�  r‹  r3  r1  r2  r5  r�  r�  rŽ  r„  r  s   @rV   r†  r†  >  s%   ø† ñõHò(A)òF#÷J	ð 	rX   r†  c                   ón   ^ • \ rS rSrSrU 4S jr\S 5       r\S 5       r\S 5       r	S r
S rS	 rS
rU =r$ )ÚT5TrainStepiÎ  zU
T5 train step class.

Args:
    args (`argparse.Namespace`): Megatron-LM arguments.
c                 ó  >• [         TU ]  S5        U R                  XR                  5      U l        U R                  U5      U l        U R                  5       U l        UR                  (       d  S U l
        g SSKJn  X0l
        g )Nr¼  r   )ÚSeq2SeqLMOutput)rú   rz   r%  ro   r1  r+  r2  r(  r3  r4  r5  r6  r¾  )rv   re   rS   r¾  rû   s       €rV   rz   ÚT5TrainStep.__init__Ö  se   ø€ Ü‰Ñ˜Ô'Ø×,Ñ,¨[×:TÑ:TÓUˆŒØ×+Ñ+¨KÓ8ˆŒØ ×6Ñ6Ó8ˆÔØ×%×%Ø&*ˆDÕ#åEà&5Õ#rX   c                 ó\   • U R                  S5      nU R                  S5      nX-  nUS:  nU$ )Nr   r   ç      à?)Ú	unsqueeze)rQ  Úattention_mask_b1sÚattention_mask_bs1Úattention_mask_bssÚextended_attention_masks        rV   Úattn_mask_postprocessÚ!T5TrainStep.attn_mask_postprocessâ  s@   € ð ,×5Ñ5°aÓ8Ðà+×5Ñ5°aÓ8Ðà/ÑDÐà"4°sÑ":ÐØ&Ð&rX   c                 óf   • [         R                  " [         R                  " SX 4US95      nUS:  nU$ ©Nr   ©rÉ   rÁ  )rÌ   ÚtrilÚones)r˜   rÉ   rQ  s      rV   Úget_decoder_maskÚT5TrainStep.get_decoder_maskï  s1   € äŸš¤E§J¢J°°:Ð/JÐSYÑ$ZÓ[ˆØ'¨#Ñ-ˆØÐrX   c                 ó„   • U R                   u  p4U R                  S5      n[        R                  " X1S4US9nXe-  nUS:  nU$ rÊ  )rŸ  rÂ  rÌ   rÍ  )	rQ  Údec_seq_lengthrÉ   rØ   Ú_rÃ  rÄ  rÅ  rÆ  s	            rV   Úget_enc_dec_maskÚT5TrainStep.get_enc_dec_maskõ  sS   € à&×,Ñ,‰ˆ
ð ,×5Ñ5°aÓ8Ðä"ŸZšZ¨ÀQÐ(GÐPVÑWÐØ/ÑDÐØ"4°sÑ":ÐØ&Ð&rX   c                 óÖ   • S nS nUR                   R                  R                  b   UR                   R                  R                  $ U(       a	   SSKJn  U$ U$ ! [
         a     U$ f = f)Nc                 óN  • / SQn[         R                  nU b  [        U 5      nOSn[        R                  " XU5      nUS   R                  5       nUS   R                  5       nUS   R                  5       nUS   R                  5       nUS   S:  n	US	   S:  n
US
   S:  nXVX‡XšU4$ )r:  )Útext_encÚtext_decr=  r?  Úenc_maskÚdec_maskÚenc_dec_maskNr×  rØ  r=  r?  rÙ  rÁ  rÚ  rÛ  rA  )rÑ   rG  rH  ræ   rI  Ú
tokens_encÚ
tokens_decr=  r?  rÙ  rÚ  rÛ  s               rV   rM  Ú6T5TrainStep.get_batch_func.<locals>.get_batch_megatron  sÇ   € ò kˆDÜ—{‘{ˆHð Ñ(Ü˜MÓ*‘à�Ü$×3Ò3°DÀÓIˆFð   
Ñ+×0Ñ0Ó2ˆJØ 
Ñ+×0Ñ0Ó2ˆJØ˜HÑ%×*Ñ*Ó,ˆFØ˜{Ñ+×1Ñ1Ó3ˆIà˜jÑ)¨CÑ/ˆHØ˜jÑ)¨CÑ/ˆHØ! .Ñ1°CÑ7ˆLà¨9¸hÐR^Ð^Ð^rX   c                 ó2  • [        U 5      n[        U[        R                  R	                  5       5      nUS   R                  5       nUS   R                  5       nUS:g  R                  [        R                  5      nSU;   a  US   R                  5       nOkUR                  UR                  UR                  [        R
                  S9nUSSS24   R                  5       USS	S24'   S
US'   UR                  US:H  S
5        [        R                  US   R                  5       5      n[        R                  UR                  S	   UR                  5      n[        R!                  US   R                  5       UR                  S	   UR                  5      nX%XCXgU4$ )r:  rP  r=  rS  Údecoder_input_ids)rÉ   rÈ   .Nra  r   r   ).r   rQ  )rC  r   rÌ   rU  rV  rE  rW  rF  Ú	new_zerosrŸ  rÉ   ÚcloneÚmasked_fill_r¼  rÇ  rÎ  rÓ  )	rÑ   ræ   rÜ  r=  r?  rÝ  rÙ  rÚ  rÛ  s	            rV   rX  Ú9T5TrainStep.get_batch_func.<locals>.get_batch_transformer  st  € ä˜Ó&ˆDÜ! $¬¯
©
×(AÑ(AÓ(CÓDˆDà˜kÑ*×/Ñ/Ó1ˆJØ˜(‘^×(Ñ(Ó*ˆFØ 4™×+Ñ+¬E¯K©KÓ8ˆIØ" dÓ*Ø!Ð"5Ñ6×;Ñ;Ó=‘
à#×-Ñ-¨f¯l©lÀ6Ç=Á=ÔX]×XbÑXbÐ-Ðc�
Ø&,¨S°#°2°#¨XÑ&6×&<Ñ&<Ó&>�
˜3 ¡˜7Ñ#Ø%&�
˜6Ñ"Ø×'Ñ'¨
°dÑ(:¸AÔ>Ü"×8Ñ8¸Ð>NÑ9O×9TÑ9TÓ9VÓWˆHÜ"×3Ñ3°J×4DÑ4DÀQÑ4GÈ×IZÑIZÓ[ˆHÜ&×7Ñ7ØÐ%Ñ&×+Ñ+Ó-¨z×/?Ñ/?ÀÑ/BÀJ×DUÑDUóˆLð ¨9¸hÐR^Ð^Ð^rX   r   rZ  )r[   r\   r[  r¥   r1  r¦   r\  s         rV   r%  ÚT5TrainStep.get_batch_func  ss   € ò	_ò2	_ð. ×Ñ×/Ñ/×IÑIÑUØ×$Ñ$×7Ñ7×QÑQÐQÞ ðå1à Ð ð
 )Ð(øô	 ó ØØ%Ð%ðúr^  c                 óŽ   • S nUR                   R                  R                  b   UR                   R                  R                  $ U$ )Nc                 óà   • UR                  5       n[        R                  " UR                  S5      U R	                  S5      -  5      U R                  5       -  nUn[        U/5      nUSUS   04$ )Nra  rc  r   )rF  rÌ   rd  re  rf  r9   )r?  ri  rj  rl  rn  ro  s         rV   r2  Ú,T5TrainStep.get_loss_func.<locals>.loss_funcA  sh   € Ø$×*Ñ*Ó,ˆHÜ—i’i §¡¨bÓ 1°I×4EÑ4EÀbÓ4IÑ IÓJÈYÏ]É]Ë_Ñ\ˆGàˆDÜGÈÈ	ÓRˆOà˜) _°QÑ%7Ð8Ð8Ð8rX   rx  )rv   re   r2  s      rV   r+  ÚT5TrainStep.get_loss_func@  s?   € ò	9ð ×Ñ×/Ñ/×DÑDÑPØ×$Ñ$×7Ñ7×LÑLÐLØÐrX   c                 ó   ^ • U 4S jnU$ )Nc           
      ót   >• T
R                  U 5      u  p#pEpgnU" X#XgUSUS9n	U	[        T
R                  U5      4$ )r}  Nr~  r€  )rÑ   rU   rÜ  rÝ  r?  rL  rÙ  rÚ  rÛ  ri  rv   s             €rV   r3  Ú7T5TrainStep.get_forward_step_func.<locals>.forward_stepO  sU   ø€ ð ^b×]kÑ]kØó^ÑZˆJ I¸(Èlñ "Ø¨¸LÐX\ÐhqñˆMð !¤'¨$¯.©.¸)Ó"DÐDÐDrX   r’   r¹  s   ` rV   r(  Ú!T5TrainStep.get_forward_step_funcN  s   ø€ õ	Eð ÐrX   rƒ  )rµ   r¶   r·   r¸   r¹   rz   ÚstaticmethodrÇ  rÎ  rÓ  r%  r+  r(  rº   r
  r  s   @rV   r¼  r¼  Î  s^   ø† ñõ
6ð ñ
'ó ð
'ð ñó ðð
 ñ	'ó ð	'ò=)ò~÷ð rX   r¼  c                  óÄ   • [        5       n [        S S S 5        U R                  S:X  a  [        SU R                   S35        [        U R                  U R                  5        g )Nr   z> setting random seeds to z ...)r   r.   rL   rM   r�   r/   Údata_parallel_random_init)rS   s    rV   Úfinish_mpu_initrñ  _  sL   € ä‹:€Dä˜D $¨Ô-ð ‡y�y�Aƒ~ÜÐ*¨4¯9©9¨+°TÐ:Ô;Ü�T—Y‘Y × >Ñ >Õ?rX   c                 óV  • Uc  0 nU R                  S5        [        R                  R                  5       (       d   S5       e[	        USS9nUR                  5        HM  u  pE[        X4S 5      b/  UR                  S:X  a  [        SU S[        X45       SU SU 3SS	9  [        X4U5        MO     UR                  (       d  UR                  S
S5      (       a  UR                  c   S5       e[        U5        [        U5        [        USS9  [        5         [!        5         [#        5         [%        5         ['        5       n[        USS 5      c  [)        UR*                  U5      Ul        UR.                  S:X  a)  UR0                  (       a  UR2                  S:X  a  SUl        OSUl        SUl        g )NzInitializing Megatron-LMzMegatron requires CUDA.T)Úignore_unknown_argsr   z*WARNING: overriding default arguments for r~   r   )ÚflushÚuse_checkpoint_argsFz/--use-checkpoints-args requires --load argument)Úbuild_tokenizerÚpadded_vocab_sizer=   r   )rM   rÌ   rU  Úis_availabler%   r€   r�   rL   r‚   rõ  ÚgetÚloadr'   r&   r*   rñ  r-   r,   r0   r   r2   Úorig_vocab_sizer÷  rN   rK   rP   rO   Ú	iteration)re   Úextra_args_providerÚargs_defaultsrS   rƒ   r„   s         rV   Ú
initializerÿ  l  sŽ  € ØÑØˆØ×ÑÐ0Ô1Ü�:‰:×"Ñ"×$Ñ$Ð?Ð&?Ó?Ð$ô Ð)¸tÑD€Dð $×)Ñ)Ö+‰
ˆÜ�4˜dÓ#Ñ/Ø�y‰y˜A‹~ÜØ@ÀÀÀQÄwÈtÓGYÐFZÐZ`ÐadÐ`eÐefÐglÐfmÐnØòô 	�˜5Ö!ñ ,ð ×× =×#4Ñ#4Ð5JÈE×#RÑ#RØ�y‰yÑ$ÐWÐ&WÓWÐ$Ü! $Ô'ä�$Ôô ˜¨uÒ5ô Ôô Ôô Ôô ÔÜ‹:€DÜˆtÐ(¨$Ó/Ñ7Ü!9¸$×:NÑ:NÐPTÓ!UˆÔØ×Ñ˜vÓ%¨$×*?×*?ÀDÇOÁOÐWXÓDXØ $ˆÕà %ˆÔØ€D…NrX   c                   óh   ^ • \ rS rSrSrU 4S jrS rS rS rS r	S r
S	 rS
 rS rS rS rSrU =r$ )ÚMegatronEngineiž  zâ
Megatron-LM model wrapper

Args:
    accelerator (:class:`~accelerate.Accelerator`): The accelerator object to use.
    model: Megatron-LM model
    optimizer: Megatron-LM optimizer
    lr_scheduler: Megatron-LM lr scheduler
c                 óâ  >• [         TU ]  5         X l        US   U l        X0l        X@l        [        5       nUR                  R                  R                  bK  UR                  R                  R                  " U40 UR                  R                  R                  D6U l        O{UR                  S:X  a  [        X5      U l        OZUR                  S:X  a  [        X5      U l        O9UR                  S:X  a  [        X5      U l        O[!        SUR                   35      eSU R                  l        0 U l        0 U l        SU l        SU l        SU l        S U l        UR0                  b  [3        5         g g )Nr   r=   rE   rG   rJ   FT)rú   rz   ÚmoduleÚ
base_modelrg   rZ   r   r[   r\   Úcustom_train_step_classÚcustom_train_step_kwargsÚtrain_step_handlerrN   r.  r†  r¼  rR   r  Útotal_loss_dictÚeval_total_loss_dictrü  Úreport_memory_flagÚ$num_floating_point_operations_so_farÚmodule_configÚtensorboard_dirr1   )rv   re   rU   rg   rZ   rS   rû   s         €rV   rz   ÚMegatronEngine.__init__©  sD  ø€ Ü‰ÑÔØŒØ ™(ˆŒØ"ŒØ"ŒÜ‹zˆØ×Ñ×/Ñ/×GÑGÑSØ&1×&7Ñ&7×&JÑ&J×&bÒ&bØñ'Ø#×)Ñ)×<Ñ<×UÑUñ'ˆDÕ#ð ×!Ñ! VÓ+Ü&3°KÓ&FˆDÕ#Ø×!Ñ! UÓ*Ü&2°;Ó&EˆDÕ#Ø×!Ñ! TÓ)Ü&1°+Ó&DˆDÕ#äÐ7¸×8LÑ8LÐ7MÐNÓOÐOØ&+ˆ�‰Ô#ð  "ˆÔØ$&ˆÔ!ØˆŒØ"&ˆÔØ45ˆÔ1Ø!ˆÔØ×ÑÑ+Ü%Õ'ð ,rX   c                 óÚ  ^ ^• [        5       n[        T R                  S   5      nT R                  R                  Ul        [        T R                  S   [        5      (       aæ  UR                  (       aÕ  UR                  b   S5       eT R                   Vs/ s H  o3R                  PM     snUl	        [        T R                  5      S:X  a  UR                  S   Ul	        UR                  (       aX  T R                   Vs/ s H  o3R                  PM     snUl        [        T R                  5      S:X  a  UR                  S   Ul        UR                  (       ax  UR                   (       ag  [#        [        T R                  5      5       V^s/ s H
  mUU 4S jPM     snUl        [        T R                  5      S:X  a  UR$                  S   Ul        [&        Ul        U$ s  snf s  snf s  snf )Nr   z‡When overlap_grad_reduce is True, config.no_sync_func must be None; a custom no_sync_func is not supported when overlapping grad-reducer   c                 ó<   >• TR                   R                  TU 5      $ r¿   )rg   Úfinish_param_sync)ÚxÚmodel_indexrv   s    €€rV   Ú<lambda>Ú2MegatronEngine.get_module_config.<locals>.<lambda>Û  s   ø€ ˜$Ÿ.™.×:Ñ:¸;ÈÔJrX   )r   r   r  rg   Ú
scale_lossÚgrad_scale_funcr“   ÚLocalDDPÚoverlap_grad_reduceÚno_sync_funcÚno_syncrc   Údelay_grad_reduceÚstart_grad_syncÚgrad_sync_funcÚoverlap_param_gatherÚdelay_param_gatherr«   Úparam_sync_funcr   Úfinalize_model_grads_func)rv   rS   r>   Úmodel_chunkr  s   `   `rV   Úget_module_configÚ MegatronEngine.get_module_configÈ  s‰  ù€ Ü‹zˆÜ! $§+¡+¨a¡.Ó1ˆà!%§¡×!:Ñ!:ˆÔÜ�d—k‘k !‘n¤h×/Ñ/°D×4L×4LØ×&Ñ&Ñ.ð ðVóÐ.ð KOÏ+Ê+Ó"VÊ+¸;×#6Ô#6É+Ñ"VˆFÔÜ�4—;‘;Ó 1Ó$Ø&,×&9Ñ&9¸!Ñ&<�Ô#Ø×%×%ØX\×XcÒXcÓ(dÒXcÈ×)DÔ)DÑXcÑ(d�Ô%Ü�t—{‘{Ó# qÓ(Ø,2×,AÑ,AÀ!Ñ,D�FÔ)Ø×$×$¨×)@×)@ä^cÔdgÐhl×hsÑhsÓdtÔ^uô&Ú^uÈ{×JÐJÑ^uñ&ˆFÔ"ô �4—;‘;Ó 1Ó$Ø)/×)?Ñ)?ÀÑ)B�Ô&Ü+?ˆÔ(Øˆùò #Wùò )eùò&s   ÂGÃ>G#ÆG(c                 ó®   • U R                    H  nUR                  5         M     U R                  c  U R                  5       U l        U R	                  5         g r¿   )r  Útrainr  r$  Úlog_eval_results©rv   Úmodel_modules     rV   r'  ÚMegatronEngine.trainâ  sG   € Ø ŸKœKˆLØ×ÑÖ ñ (ð ×ÑÑ%Ø!%×!7Ñ!7Ó!9ˆDÔà×ÑÕrX   c                 ó�   • U R                    H  nUR                  5         M     U R                  c  U R                  5       U l        g g r¿   )r  Úevalr  r$  r)  s     rV   r-  ÚMegatronEngine.evalë  s@   € Ø ŸKœKˆLØ×ÑÖñ (ð ×ÑÑ%Ø!%×!7Ñ!7Ó!9ˆDÕð &rX   c                 óŒ  • [        5       n/ n[        U5      S:”  a„  UR                  S:”  aq  [        SUR                  5       HV  nUR	                  UR                  5        VVs0 s H&  u  pVXVXBR                  -  US-   UR                  -   _M(     snn5        MX     OU/n[        U R                  5      S:”  ad  [        U5      S:”  a:  [        [        U R                  5      5       Vs/ s H  n[        U5      PM     snnU$ S /[        U R                  5      -  nU$ [        U5      S:”  a  [        U5      OS nU$ s  snnf s  snf )Nr   r   )	r   rc   rä   r«   r­   r€   rã   r  Úiter)	rv   Ú
batch_datarS   Údata_chunksr²   rò   ÚvrÒ  Úbatch_data_iterators	            rV   Úget_batch_data_iteratorÚ&MegatronEngine.get_batch_data_iteratorò  sC  € Ü‹zˆØˆÜˆz‹?˜QÓØ×%Ñ%¨Ó)Ü˜q $×"8Ñ"8Ö9�AØ×&Ñ&ð )3×(8Ñ(8Ô(:ôâ(:¡ ð  ×%:Ñ%:Ñ!:¸aÀ!¹eÀt×G\ÑG\Ñ=\Ð]Ò]Ù(:òöò :ð  *˜l�äˆt�{‰{Ó˜aÓô �z“? QÓ&ô -2´#°d·k±kÓ2BÔ,CÓDÒ,C q”�kÖ"Ñ,CÑDð  ð #Ð"ð	 �Vœc $§+¡+Ó.Ñ.ð  ð #Ð"ô 8;¸:³ÈÓ7J¤$ {Ô"3ÐPTÐØ"Ð"ùó!ùò Es   Á#-D;Ã(Ec           
      ó  • U R                  U5      n[        U R                  R                  UU R                  U R
                  U R                  U R                  [        5       S9u  p4    pVnUS:H  U R
                  l	        X4Xg4$ )z`
Training step for Megatron-LM

Args:
    batch_data (:obj:`dict`): The batch data to train on.
)Úforward_step_funcrÑ   rU   rg   Úopt_param_schedulerr>   Úforward_backward_funcr   )
r5  r7   r  r3  r  rg   rZ   r  r   r  )rv   r1  r4  Úloss_reducedr  rÒ  Ú	grad_normÚnum_zeros_in_grads           rV   r7   ÚMegatronEngine.train_step  s€   € ð #×:Ñ:¸:ÓFÐäLVØ"×5Ñ5×BÑBØ-Ø—+‘+Ø—n‘nØ $§¡Ø×%Ñ%Ü";Ó"=ñM
ÑIˆ A q¨!Ð8Ið '3°aÑ&7ˆ�‰Ô#à¨9ÐGÐGrX   c           
      óÚ  • [        5       nU R                  U5      n[        5       nU" U R                  R                  UU R
                  [        5       UR                  UR                  SS9nUR                  S:¼  a  [        R                  R                  5         U=R                  [        R                  " 5       UR                  -  [        5       -  -  sl        [        R                   " SS9(       as  0 nUS    Hf  nU Vs/ s H  oˆU   PM	     n	n[#        U	S   R$                  5      S:X  a  ['        U	5      [#        U	5      -  Xg'   MN  [        R(                  " U	5      Xg'   Mh     U$ 0 $ s  snf )ze
Evaluation step for Megatron-LM

Args:
    batch_data (:obj:`dict`): The batch data to evaluate on.
T)r8  rÑ   rU   Únum_microbatchesr˜   rã   Úforward_onlyr   )Úignore_virtualr   )r   r5  r   r  r3  r  r   r˜   rã   Úempty_unused_memory_levelrÌ   rU  Úempty_cacherï   r   rê   Úis_pipeline_last_stagerc   rŸ  rd  r   )
rv   r1  rS   r4  r:  Ú
loss_dictsr;  rƒ   r  Úlosses_reduced_for_keys
             rV   Ú	eval_stepÚMegatronEngine.eval_step#  sC  € ô ‹zˆØ"×:Ñ:¸:ÓFÐÜ 9Ó ;ÐÙ*Ø"×5Ñ5×BÑBØ-Ø—+‘+Ü1Ó3Ø—‘Ø!×2Ñ2Øñ
ˆ
ð ×)Ñ)¨QÓ.Ü�J‰J×"Ñ"Ô$à×#Ò#Ü×,Ò,Ó.°×1FÑ1FÑFÔI]ÓI_Ñ_ñ	
Õ#ô ×%Ò%°T×:àˆLØ! !”}�Ù:DÓ)Eº*°Q¨C¬&¹*Ð&Ð)EÜÐ-¨aÑ0×6Ñ6Ó7¸1Ó<Ü(+Ð,BÓ(CÄcÐJ`ÓFaÑ(a�LÓ%ä(-¯ªÐ5KÓ(L�LÓ%ñ %ð  ÐØˆ	ùò *Fs   ÄE(c                 óÄ  • [        5       nU R                  S   R                  (       Ga9  U R                  " S
0 UD6u  p4pVU =R                  S-  sl        [
        R                  " 5       UR                  -  [        5       -  nU=R                  U-  sl	        U =R                  [        X'5      -  sl
        UR                  b¡  U R                  R                  5       R                  5       nS n	UR                   (       a  [#        U R$                  5      n	['        UU R(                  U R                  R*                  S   S   U R                  UU R,                  UUU	U5
      U l        OâU R.                  " S
0 UD6nUR                  bÃ  U H½  n
U R0                  R3                  U
[4        R6                  R9                  S/5      5      X:   -   U R0                  U
'   U R0                  R3                  U
S-   [4        R6                  R9                  S/5      5      [4        R6                  R9                  S/5      -   U R0                  U
S-   '   M¿     [4        R:                  " S[4        R6                  R=                  5       S9nU H'  n
[?        X:   R@                  5      S:X  d  M   X³U
   -  nM)     S nSU;   a  US   nU RB                  RD                  b  U RB                  RE                  X¼S	9$ U$ )Nr   r   Úlrg        Ú
_num_itersg      ð?rË  rt  )rn  rt  r’   )#r   r  Útrainingr7   rü  r   rê   rã   r   rî   r  r5   r  rg   Úget_loss_scaleÚitemÚlog_params_normr:   rU   r8   r  Úparam_groupsr
  rH  r	  rù  rÌ   rU  ÚFloatTensorrÍ   rV  rc   rŸ  r  r5  )rv   r1  rS   Ú	loss_dictr  r<  r=  rØ   Ú
loss_scaleÚparams_normrƒ   rn  rt  s                rV   ÚforwardÚMegatronEngine.forwardK  s�  € ô ‹zˆØ�;‰;�q‰>×"×"Ð"ØDHÇOÂOÑDaÐV`ÑDaÑAˆI YØ�NŠN˜aÑ�NÜ×9Ò9Ó;¸d×>SÑ>SÑSÔVjÓVlÑlˆJØ×'Ò'¨:Ñ5Õ'Ø×5Ò5Ô9VÐW[Ó9hÑhÕ5Ø×#Ñ#Ñ/à!Ÿ^™^×:Ñ:Ó<×AÑAÓC�
Ø"�Ø×'×'Ü"5°d·j±jÓ"A�KÜ*6ØØ×(Ñ(Ø—N‘N×/Ñ/°Ñ2°4Ñ8Ø—N‘NØØ×+Ñ+Ø ØØØ%ó+�Ô'øð ŸšÑ4¨Ñ4ˆIØ×#Ñ#Ñ/Û$�Cà×1Ñ1×5Ñ5°c¼5¿:¹:×;QÑ;QÐSVÐRWÓ;XÓYÐ\eÑ\jÑjð ×-Ñ-¨cÑ2ð EI×D]ÑD]×DaÑDaØ˜lÑ*¬E¯J©J×,BÑ,BÀCÀ5Ó,IóEäŸ
™
×.Ñ.°¨uÓ5ñE6�D×-Ñ-¨c°LÑ.@ÓAñ	 %ô �|Š|˜C¬¯
©
×(AÑ(AÓ(CÑDˆÛˆCÜ�9‘>×'Ñ'Ó(¨AÕ-Ø #™Ñ&’ñ ð ˆØ�yÓ Ø˜xÑ(ˆFØ×"Ñ"×5Ñ5ÑAØ×*Ñ*×=Ñ=À4Ð=ÐWÐWØˆrX   c                 ó&  • [        5       nUR                  b  U R                  S:X  a  g [        5       n[        5       nSU R                   S3nU R                   Hù  nUR                  S5      (       a  M  U R                  U   U R                  US-      -  nX4 SU S3-  n[        R                  " [        SUR                  5       5      5      nUR                  (       a
  X4 SU S3-  nU(       d  M™  UR                  U S3UR                  5       U R                  5        UR                  (       d  MÚ  UR                  U S	3X`R                  5        Mû     [        U5      S
-   n[        SU-  5        [        U5        [        SU-  5        0 U l        g )Nr   zvalidation loss at iteration z | rL  z value: é   z PPL: z validationz validation pplr   Ú-)r   r  rü  r   r	  ÚendswithÚmathÚexpÚminrO  rK   Ú
add_scalarrc   r!   )rv   rS   ÚwriterÚstringrƒ   r„   ÚpplÚlengths           rV   r(  ÚMegatronEngine.log_eval_resultsŠ  sa  € Ü‹zˆØ×ÑÑ'¨4¯>©>¸QÓ+>ØÜ‹zˆÜ'Ó)ˆØ0°·±Ð0@ÀÐDˆØ×,Ô,ˆCØ�|‰|˜L×)Ñ)ÙØ×-Ñ-¨cÑ2°T×5NÑ5NÈsÐUaÑOaÑ5bÑbˆEØ˜˜X e W¨CÐ0Ñ0ˆFÜ—(’(œ3˜r 5§:¡:£<Ó0Ó1ˆCØ×$×$Ø˜E ¨ u¨CÐ0Ñ0�ßˆvØ×!Ñ! S E¨Ð"5°u·z±z³|ÀTÇ^Á^ÔTØ×(×(Ñ(Ø×%Ñ%¨¨¨_Ð&=¸sÇNÁNÖSñ -ô �V“˜q‘ˆÜ˜˜f™Ô%Ü˜ÔÜ˜˜f™Ô%Ø$&ˆÕ!rX   c                 ó:  • U R                  5         [        5       nXl        [        R                  R                  5         [        U R                  U R                  U R                  U R                  U R                  S9  [        R                  R                  5         g )N)r  )r(  r   ÚsaverÌ   rÏ   Úbarrierr)   rü  r  rg   rZ   r  )rv   Ú
output_dirrS   s      rV   r)   ÚMegatronEngine.save_checkpoint¤  sm   € Ø×ÑÔÜ‹zˆØŒ	Ü×Ñ×!Ñ!Ô#ÜØ�N‰NØ�K‰KØ�N‰NØ�N‰NØ15×1ZÑ1Zò	
ô 	×Ñ×!Ñ!Õ#rX   c                 ó¤  • [        5       nXl        SUl        SUl        [        R
                  R                  5         [        U R                  U R                  U R                  5      u  p4[        R
                  R                  5         X0l        X@l        UR                  (       a,  U R                  S:X  a  U R                  R                  5         g g g r  )r   rú  rî   rï   rÌ   rÏ   rg  r(   r  rg   rZ   rü  r  Úfp16Úreload_model_params)rv   Ú	input_dirrS   rü  r  s        rV   r(   ÚMegatronEngine.load_checkpoint²  s™   € Ü‹zˆØŒ	Ø&'ˆÔ#Ø&'ˆÔ#Ü×Ñ×!Ñ!Ô#Ü:IÈ$Ï+É+ÐW[×WeÑWeÐgk×guÑguÓ:vÑ7ˆ	Ü×Ñ×!Ñ!Ô#Ø"ŒØ4XÔ1Ø�9�9˜Ÿ™¨1Ó,Ø�N‰N×.Ñ.Õ0ð -ˆ9rX   )r  r	  rü  r  r  r  rg   r
  rZ   r  r  )rµ   r¶   r·   r¸   r¹   rz   r$  r'  r-  r5  r7   rH  rV  r(  r)   r(   rº   r
  r  s   @rV   r  r  ž  sG   ø† ñõ(ò>ò4 ò:ò#ò2Hò0&òP=ò~'ò4$÷1ð 1rX   r  c                 ó   • [        U 5      $ )z„
Average losses across data parallel group.

Args:
    losses (List[Tensor]): List of losses to average across data parallel group.
)r9   )r°  s    rV   Ú%avg_losses_across_data_parallel_grouprp  Á  s   € ô 5°VÓ<Ð<rX   c                 ó   • S n[        XSS9$ )zÞ
Recursively gather tensor in a nested list/tuple/dictionary of tensors from data parallel ranks.

Args:
    tensor (nested list/tuple/dictionary of `torch.Tensor`):
        The data to gather across data parallel ranks.

c                 ó˜  • U R                   S:X  a  U R                  5       S    n [        [        R                  R                  [        R                  " 5       S95       Vs/ s H  n[        R                  " U 5      PM     nn[        R                  R                  X [        R                  " 5       S9  [        R                  " USS9$ s  snf )Nr   rÊ   rœ  )Úndimrâ  r«   rÌ   rÏ   Úget_world_sizer   Úget_data_parallel_groupÚ
empty_likeÚ
all_gatherr¨  )rÍ   rÒ  Úoutput_tensorss      rV   Ú_gpu_gather_oneÚ;gather_across_data_parallel_groups.<locals>._gpu_gather_oneÖ  s§   € Ø�;‰;˜!ÓØ—\‘\“^ DÑ)ˆFô œ5×,Ñ,×;Ñ;Ä#×B]ÒB]ÓB_Ð;Ð`Ôaó
âa�ô ×Ò˜VÖ$Ùað 	ð 
ô 	×Ñ×$Ñ$ ^Ä3×C^ÒC^ÓC`Ð$ÑaÜ�yŠy˜¨QÑ/Ð/ùò
s   Á CT)Úerror_on_other_type)r   )rÍ   ry  s     rV   Ú"gather_across_data_parallel_groupsr|  Ì  s   € ò0ô ˜_È$ÑOÐOrX   )TTTT)NN)orp   r\  r®  Úabcr   Ú	functoolsr   rÌ   Útorch.nn.functionalÚnnÚ
functionalrg  Útorch.nnr   r   r   rg   r	   rZ   r
   Úimportsr   Ú
operationsr   r   Úmegatron.corer   r   Úmegatron.core.distributedr   r  r   Úmegatron.core.enumsr   Ú)megatron.core.num_microbatches_calculatorr   Úmegatron.core.optimizerr   Úmegatron.core.parallel_stater   r   Úmegatron.core.pipeline_parallelr   Úmegatron.core.utilsr   Ú"megatron.legacy.data.dataset_utilsr   Úmegatron.legacy.modelr   r   Ú$megatron.legacy.model.classificationr   Úmegatron.trainingr   r   r    r!   Úmegatron.training.argumentsr"   r#   r$   r%   r&   Úmegatron.training.checkpointingr'   r(   r)   Úmegatron.training.global_varsr*   Úmegatron.training.gpt_buildersr+   Úmegatron.training.initializer,   r-   r.   r/   r0   r1   Ú%megatron.training.tokenizer.tokenizerr2   Úmegatron.training.trainingr3   r4   r5   r6   r7   r8   Úmegatron.training.utilsr9   r:   r;   rW   rj   rl   rÔ   r×   rõ   r_   r  r  r`   r   r.  r†  r¼  rñ  rÿ  ÚModuler  rp  r|  r’   rX   rV   Ú<module>rš     sf  ðó Û Û 	Ý Ý ã ß Ð ß AÑ Aå ,Ý ,Ý -ß 9ñ ×Ñß2ÝMÝ>Ý-ÝNÝ>ßpÝIÝ4ÝRß8ÝC÷ó ÷õ ÷ lÑkÝBÝ:÷÷ õ O÷÷ ÷ñ ô.òb'÷8jLñ jLòZò$>LôD+Ð!5ô +ò b÷ñ ô.Ð!5ô òô˜ô ô"KÐ%ô Kô\MÐ$ô Mô`NÐ#ô Nòb	@ô/ôd_1�U—X‘X—_‘_ô _1òF	=óPrX   