ó
    Eñi   ã                   ó”  • S SK Jr  SSKJrJrJr  SSKJrJrJ	r	J
r
JrJrJrJrJrJrJrJrJrJrJrJrJrJr  SSKJrJrJrJrJrJrJ r J!r!J"r"J#r#J$r$J%r%J&r&J'r'J(r(J)r)J*r*J+r+J,r,J-r-J.r.J/r/J0r0J1r1J2r2J3r3J4r4J5r5J6r6J7r7J8r8J9r9J:r:  SSK;J<r<J=r=J>r>J?r?J@r@JArAJBrBJCrCJDrDJErEJFrFJGrGJHrHJIrIJJrJ  SSKKJLrLJMrMJNrNJOrOJPrPJQrQJRrRJSrSJTrTJUrUJVrVJWrWJXrXJYrYJZrZJ[r[J\r\J]r]J^r^J_r_J`r`JaraJbrbJcrcJdrdJereJfrfJgrgJhrhJiriJjrjJkrkJlrlJmrmJnrnJoroJprpJqrqJrrrJsrsJtrtJuruJvrvJwrwJxrxJyryJzrzJ{r{J|r|J}r}J~r~JrJ€r€J�r�J‚r‚Jƒrƒ  SSK„J…r…J†r†J‡r‡JˆrˆJ‰r‰JŠrŠJ‹r‹JŒrŒJ�r�JŽrŽJ�r�J�r�J‘r‘J’r’J“r“J”r”J•r•J–r–J—r—J˜r˜J™r™JšršJ›r›Jœrœ  SS	K�JžržJŸrŸJ r J¡r¡J¢r¢J£r£J¤r¤  SS
K¥J¦r¦J§r§J¨r¨J©r©JªrªJ«r«J¬r¬J­r­J®r®J¯r¯J°r°J±r±J²r²J³r³J´r´JµrµJ¶r¶J·r·J¸r¸J¹r¹JºrºJ»r»J¼r¼J½r½J¾r¾J¿r¿  SSKÀJÁrÁJÂrÂ  \X" 5       (       a  SSKÃJÄrÄJÅrÅJÆrÆJÇrÇJÈrÈJÉrÉJÊrÊJËrË  SSKÌJÍrÍJÎrÎ  SSKÏJÐrÐJÑrÑJÒrÒJÓrÓJÔrÔJÕrÕJÖrÖJ×r×JØrØJÙrÙJÚrÚJÛrÛJÜrÜJÝrÝ  SSKÞJßrßJàràJáráJârâJãrãJäräJårå  SSKæJçrçJèrèJéréJêrêJërëJìrìJírí  \a" 5       (       a*  SSKæJîrîJïrïJðrðJñrñ  SSKæJòró  SSKæJôrõ  SSKæJör÷  SSKæJørù  SSKæJúrû  SSKüJýrýJþrþ  SSKÿGJ Gr GJGrGJGrGJGrGJGrGJGrGJGrGJGrGJGrGJ	Gr	GJ
Gr
GJGrGJGrGJGrGJGrGJGrGJGrGJGr  SSGKGJGrGJGrGJGr  SSGKGJGr  SSGKGJGr  SSGKGJGrGJGrGJGrGJGr  g)é   )ÚParallelismConfigé   )Úconvert_model_to_fp8_aoÚ#filter_first_and_last_linear_layersÚhas_ao_layers)Ú(MITA_PROFILING_AVAILABLE_PYTORCH_VERSIONÚ
MODEL_NAMEÚOPTIMIZER_NAMEÚPROFILE_PATTERN_NAMEÚRNG_STATE_NAMEÚSAFE_MODEL_NAMEÚSAFE_WEIGHTS_INDEX_NAMEÚSAFE_WEIGHTS_NAMEÚSAFE_WEIGHTS_PATTERN_NAMEÚSAMPLER_NAMEÚSCALER_NAMEÚSCHEDULER_NAMEÚ!TORCH_DISTRIBUTED_OPERATION_TYPESÚTORCH_LAUNCH_PARAMSÚWEIGHTS_INDEX_NAMEÚWEIGHTS_NAMEÚWEIGHTS_PATTERN_NAMEÚ'XPU_PROFILING_AVAILABLE_PYTORCH_VERSION)!ÚAORecipeKwargsÚAutocastKwargsÚBnbQuantizationConfigÚComputeEnvironmentÚCustomDtypeÚDataLoaderConfigurationÚDDPCommunicationHookTypeÚDeepSpeedPluginÚDeepSpeedSequenceParallelConfigÚDistributedDataParallelKwargsÚDistributedTypeÚDynamoBackendÚFP8RecipeKwargsÚFullyShardedDataParallelPluginÚGradientAccumulationPluginÚGradScalerKwargsÚInitProcessGroupKwargsÚKwargsHandlerÚ
LoggerTypeÚMegatronLMPluginÚMSAMPRecipeKwargsÚPrecisionTypeÚProfileKwargsÚProjectConfigurationÚRNGTypeÚSageMakerDistributedTypeÚTensorInformationÚTERecipeKwargsÚTorchContextParallelConfigÚTorchDynamoPluginÚTorchTensorParallelConfigÚTorchTensorParallelPluginÚ#add_model_config_to_megatron_parser)Úare_libraries_initializedÚcheck_cuda_fp8_capabilityÚcheck_cuda_p2p_ib_supportÚclear_environmentÚconvert_dict_to_env_variablesÚget_cpu_distributed_informationÚget_current_device_typeÚget_gpu_infoÚget_int_from_envÚparse_choice_from_envÚparse_flag_from_envÚpatch_environmentÚpurge_accelerate_environmentÚset_numa_affinityÚstr_to_bool)8Údeepspeed_requiredÚis_4bit_bnb_availableÚis_8bit_bnb_availableÚis_aim_availableÚis_bf16_availableÚ'is_bitsandbytes_multi_backend_availableÚis_bnb_availableÚis_boto3_availableÚis_clearml_availableÚis_comet_ml_availableÚis_cuda_availableÚis_datasets_availableÚis_deepspeed_availableÚis_dvclive_availableÚis_fp8_availableÚis_fp16_availableÚis_habana_gaudi1Úis_hpu_availableÚis_import_timer_availableÚis_lomo_availableÚis_matplotlib_availableÚis_megatron_lm_availableÚis_mlflow_availableÚis_mlu_availableÚis_mps_availableÚis_msamp_availableÚis_musa_availableÚis_neuron_availableÚis_npu_availableÚis_pandas_availableÚis_peft_availableÚis_pippy_availableÚis_pynvml_availableÚis_pytest_availableÚis_rich_availableÚis_sagemaker_availableÚis_schedulefree_availableÚis_sdaa_availableÚis_swanlab_availableÚis_tensorboard_availableÚis_timm_availableÚis_torch_xla_availableÚis_torchao_availableÚis_torchdata_availableÚ*is_torchdata_stateful_dataloader_availableÚis_torchvision_availableÚis_trackio_availableÚis_transformer_engine_availableÚ%is_transformer_engine_mxfp8_availableÚis_transformers_availableÚis_triton_availableÚis_wandb_availableÚis_weights_only_availableÚis_xccl_availableÚis_xpu_availableÚtorchao_required)Úalign_module_deviceÚcalculate_maximum_sizesÚcheck_device_mapÚcheck_tied_parameters_in_configÚ$check_tied_parameters_on_same_deviceÚcompute_module_sizesÚconvert_file_size_to_intÚdtype_byte_sizeÚfind_tied_parametersÚget_balanced_memoryÚget_grad_scalerÚget_max_layer_sizeÚget_max_memoryÚ#get_mixed_precision_context_managerÚhas_offloaded_paramsÚid_tensor_storageÚinfer_auto_device_mapÚis_peft_modelÚload_checkpoint_in_modelÚload_offloaded_weightsÚload_state_dictÚnamed_module_tensorsÚretie_parametersÚset_module_tensor_to_device)ÚOffloadedWeightsLoaderÚPrefixedDatasetÚextract_submodules_state_dictÚload_offloaded_weightÚoffload_state_dictÚoffload_weightÚsave_offload_index)ÚCannotPadNestedTensorWarningÚGatheredParametersÚ	broadcastÚbroadcast_object_listÚconcatenateÚconvert_outputs_to_fp32Úconvert_to_fp32Úcopy_tensor_to_devicesÚfind_batch_sizeÚfind_deviceÚgatherÚgather_objectÚget_data_structureÚ
honor_typeÚignorant_find_batch_sizeÚinitialize_tensorsÚis_namedtupleÚis_tensor_informationÚis_torch_tensorÚlistifyÚpad_across_processesÚpad_input_tensorsÚrecursively_applyÚreduceÚsend_to_deviceÚslice_tensors)Úcompare_versionsÚis_torch_version)ÚDeepSpeedEngineWrapperÚDeepSpeedOptimizerWrapperÚDeepSpeedSchedulerWrapperÚ
DummyOptimÚDummySchedulerÚHfDeepSpeedConfigÚget_active_deepspeed_pluginÚmap_pytorch_optim_to_deepspeed)Úhas_4bit_bnb_layersÚload_and_quantize_model)Ú"disable_fsdp_ram_efficient_loadingÚ!enable_fsdp_ram_efficient_loadingÚensure_weights_retiedÚfsdp2_apply_acÚfsdp2_canonicalize_namesÚfsdp2_load_full_state_dictÚfsdp2_prepare_modelÚ!fsdp2_switch_optimizer_parametersÚget_fsdp2_grad_scalerÚload_fsdp_modelÚload_fsdp_optimizerÚmerge_fsdp_weightsÚsave_fsdp_modelÚsave_fsdp_optimizer)ÚPrepareForLaunchÚ_filter_argsÚprepare_deepspeed_cmd_envÚprepare_multi_gpu_envÚprepare_sagemager_args_inputsÚprepare_simple_launcher_cmd_envÚprepare_tpu)ÚAbstractTrainStepÚBertTrainStepÚGPTTrainStepÚMegatronLMDummyDataLoaderÚMegatronLMDummySchedulerÚT5TrainStepÚ%avg_losses_across_data_parallel_group)ÚMegatronEngineÚMegatronLMOptimizerWrapperÚMegatronLMSchedulerWrapperÚ"gather_across_data_parallel_groups)Ú
initialize)Úprepare_data_loader)Ú!prepare_model_optimizer_scheduler)Úprepare_optimizer)Úprepare_scheduler)Úfind_executable_batch_sizeÚrelease_memory)Úcheck_os_kernelÚ clean_state_dict_for_safetensorsÚcompile_regionsÚcompile_regions_deepspeedÚconvert_bytesÚextract_model_from_parallelÚget_module_children_bottom_upÚget_pretty_nameÚhas_compiled_regionsÚis_compiled_moduleÚis_port_in_useÚloadÚmerge_dictsÚmodel_has_dtensorÚrecursive_getattrÚsaveÚwait_for_everyoneÚwrite_basic_config)Úset_seedÚsynchronize_rng_stateÚsynchronize_rng_states)Úinstall_xla)Útqdm)Úapply_fp8_autowrapÚcontextual_fp8_autocastÚconvert_modelÚhas_transformer_engine_layersN(  Úparallelism_configr   Úaor   r   r   Ú	constantsr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Údataclassesr   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   Úenvironmentr;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   ÚimportsrJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rs   rt   ru   rv   rw   rx   ry   rz   r{   r|   r}   r~   r   r€   r�   Úmodelingr‚   rƒ   r„   r…   r†   r‡   rˆ   r‰   rŠ   r‹   rŒ   r�   rŽ   r�   r�   r‘   r’   r“   r”   r•   r–   r—   r˜   r™   Úoffloadrš   r›   rœ   r�   rž   rŸ   r    Ú
operationsr¡   r¢   r£   r¤   r¥   r¦   r§   r¨   r©   rª   r«   r¬   r­   r®   r¯   r°   r±   r²   r³   r´   rµ   r¶   r·   r¸   r¹   rº   Úversionsr»   r¼   Ú	deepspeedr½   r¾   r¿   rÀ   rÁ   rÂ   rÃ   rÄ   ÚbnbrÅ   rÆ   Ú
fsdp_utilsrÇ   rÈ   rÉ   rÊ   rË   rÌ   rÍ   rÎ   rÏ   rÐ   rÑ   rÒ   rÓ   rÔ   ÚlaunchrÕ   rÖ   r×   rØ   rÙ   rÚ   rÛ   Úmegatron_lmrÜ   rÝ   rÞ   rß   rà   rá   râ   rã   rä   rå   ræ   rç   Úmegatron_lm_initializerè   Úmegatron_lm_prepare_data_loaderré   Ú-megatron_lm_prepare_model_optimizer_schedulerrê   Úmegatron_lm_prepare_optimizerrë   Úmegatron_lm_prepare_schedulerÚmemoryrì   rí   Úotherrî   rï   rð   rñ   rò   ró   rô   rõ   rö   r÷   rø   rù   rú   rû   rü   rý   rþ   rÿ   Úrandomr   r  r  Ú	torch_xlar  r  Útransformer_enginer  r  r  r  © ó    ÚV/home/mande/repo/quber/.venv/lib/python3.13/site-packages/accelerate/utils/__init__.pyÚ<module>r%     sê  ðõ 3ß [Ñ [÷÷ ÷ ÷ ÷ ÷("÷ "÷ "÷ "÷ "÷ "÷ "÷ "õ "÷F÷ ÷ ÷ ñ ÷"9÷ 9÷ 9÷ 9÷ 9÷ 9÷ 9÷ 9÷ 9÷ 9÷ 9÷ 9÷ 9÷ 9ó 9÷t÷ ÷ ÷ ÷ ÷ ó ÷4÷ ñ ÷÷ ÷ ÷ ÷ ÷ ÷ ÷8 9ñ ×Ñ÷	÷ 	ó 	÷ >÷÷ ÷ ÷ ÷ ÷ ñ ÷÷ ñ ñ ×Ñ÷ó õ BÝSÝoÝOÝOß >÷÷ ÷ ÷ ÷ ÷ ÷ ÷ ÷ ó ÷( L× KÐ Kß "Ð "ß Ð ÷÷ õ r#  