ó
    pyüi&  ã                   ó‚   • S SK r S SKrS SKrS SKJr  SSKJrJrJrJ	r	  \	R                  " \5      rS\4S jrS rS	S jrg)
é    N)Ú
DataLoaderé   )ÚWEIGHTS_NAMEÚPushToHubMixinÚis_torch_xla_availableÚloggingÚ
dataloaderc                 óê   • [        5       (       ac  SS KJs  Jn  [	        XR
                  5      (       d   S5       eSS KJs  Jn  UR                  UR                  5       S5      nX0R                  S'   U $ U $ )Nr   zPThe dataloader must be a `torch_xla.distributed.parallel_loader.MpDeviceLoader`.)ÚfsdpNÚinput_sharding)r   Ú%torch_xla.distributed.parallel_loaderÚdistributedÚparallel_loaderÚ
isinstanceÚMpDeviceLoaderÚtorch_xla.distributed.spmdÚspmdÚShardingSpecÚget_global_meshÚ_parallel_loader_kwargs)r	   ÚplÚxsÚsharding_specs       ÚZ/home/mande/repo/quber/.venv/lib/python3.13/site-packages/transformers/integrations/tpu.pyÚtpu_spmd_dataloaderr      sn   € Ü×Ñß:Ð:ä˜*×&7Ñ&7×8Ñ8ð 	
Ø^ó	
Ð8÷ 	0Ð/àŸ™¨×(:Ñ(:Ó(<¸nÓMˆØ?L×*Ñ*Ð+;Ñ<ØÐàÐó    c                 ó\  ^^^^^^• SSK Js  Jm  SSKJs  Jm  SSKJn   SSKJ	m  SSKJ
m  SSKJnJn  T(       a  SSKJm  SnSn[#        U S
S5      nUR$                  R'                  SU5      n	UR$                  S   S:”  a"  [(        R*                  " XAR$                  S   S9nORU	bO  [-        5       n
U	 H*  nU" X5      nUc  [/        S5      eU
R1                  U5        M,     [(        R*                  " UU
S9nUR2                  nUR$                  S   (       aJ  U R4                  R6                  (       a&  [8        R;                  S5        SU R4                  l        UUUU4S jnT(       a  U4S jnT" U UUUS9n OT" U 4UUS.UD6n S0 4U4S jjnUTl        U $ ! [          a    [!        S	5      ef = f)a  
Wraps a model with XLA Fully Sharded Data Parallelism (FSDP).

Handles both FSDP v1 (`XlaFullyShardedDataParallel`) and v2 (`SpmdFullyShardedDataParallel`),
including auto-wrap policies, gradient checkpointing, and patching `xm.optimizer_step`.

Args:
    model (`torch.nn.Module`): The model to wrap.
    args (`TrainingArguments`): The training arguments containing FSDP configuration.
    is_fsdp_xla_v2_enabled (`bool`): Whether FSDP v2 (SPMD) is enabled.

Returns:
    `torch.nn.Module`: The FSDP-wrapped model.
r   Nr   )Úget_module_class_from_name)ÚXlaFullyShardedDataParallel)Úcheckpoint_module)Úsize_based_auto_wrap_policyÚtransformer_auto_wrap_policy)ÚSpmdFullyShardedDataParallelzJMissing XLA FSDP related module; please make sure to use torch-xla >= 2.0.Ú_no_split_modulesÚtransformer_layer_cls_to_wrapÚmin_num_params)r&   z@Could not find the transformer layer class to wrap in the model.)Útransformer_layer_clsÚxla_fsdp_grad_ckptzX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Fc                 ó>   >• T(       d  TOTnU" T" U 5      /UQ70 UD6$ ©N© )ÚmÚargsÚkwargsÚ
target_clsÚFSDPÚFSDPv2r    Úis_fsdp_xla_v2_enableds       €€€€r   Úauto_wrapper_callableÚ2wrap_model_xla_fsdp.<locals>.auto_wrapper_callablet   s'   ø€ Þ%;™ÀˆJÙÑ/°Ó2ÐD°TÒD¸VÑDÐDr   c                 ó  >• SSK Jn  S n[        U [        R                  5      (       a  U nO7[        U [
        5      (       a  U S   nO[        X5      (       a  U R                  nUc  [        S5      eTR                  X1S5        g )Nr   )ÚCausalLMOutputWithPastr   zASomething went wrong, the output of the model shouldn't be `None`)r   NN)	Úmodeling_outputsr6   r   ÚtorchÚTensorÚtupleÚlogitsÚ
ValueErrorÚmark_sharding)ÚoutputÚmeshr6   Úreal_outputr   s       €r   Úshard_outputÚ)wrap_model_xla_fsdp.<locals>.shard_output{   sn   ø€ ÝAàˆKÜ˜&¤%§,¡,×/Ñ/Ø$‘Ü˜F¤E×*Ñ*Ø$ Q™i‘Ü˜F×;Ñ;Ø$Ÿm™m�àÑ"Ü Ð!dÓeÐeØ×Ñ˜[Ð0DÕEr   )rA   Úauto_wrap_policyr3   )rC   r3   c                 óZ   >• U R                   " S0 UD6nU(       a  TR                  5         U$ )Nr+   )ÚstepÚ	mark_step)Ú	optimizerÚbarrierÚoptimizer_argsÚlossÚxms       €r   Úpatched_optimizer_stepÚ3wrap_model_xla_fsdp.<locals>.patched_optimizer_stepš   s%   ø€ Ø�~Š~Ñ/ Ñ/ˆÞØ�L‰LŒNØˆr   )Útorch_xla.core.xla_modelÚcoreÚ	xla_modelr   r   r   Útrainer_pt_utilsr   Útorch_xla.distributed.fsdpr   r    Útorch_xla.distributed.fsdp.wrapr!   r"   Ú7torch_xla.experimental.spmd_fully_sharded_data_parallelr#   ÚImportErrorÚgetattrÚfsdp_configÚgetÚ	functoolsÚpartialÚsetÚ	ExceptionÚaddÚxla_fsdp_configÚconfigÚ	use_cacheÚloggerÚwarning_onceÚoptimizer_step)Úmodelr-   r2   r   r!   r"   rC   r3   Ú%default_transformer_cls_names_to_wrapÚ"fsdp_transformer_layer_cls_to_wrapÚtransformer_cls_to_wrapÚlayer_classÚtransformer_clsÚfsdp_kwargsrA   rL   r0   r1   r    rK   r   s     `             @@@@@r   Úwrap_model_xla_fsdprk   .   sÔ  ý€ ÷ *Ð)ß+Ð+å=ðhÝRÝ@÷	
ö
 "õð ÐØ ÐÜ,3°EÐ;NÐPTÓ,UÐ)Ø)-×)9Ñ)9×)=Ñ)=Ø'Ð)Nó*Ð&ð ×ÑÐ(Ñ)¨AÓ-Ü$×,Ò,Ø'×8HÑ8HÐIYÑ8Zñ
Ñð 
,Ñ	7Ü"%£%ÐÛ=ˆKÙ8¸ÓLˆOØÑ&ÜÐ bÓcÐcà'×+Ñ+¨OÖ<ñ >ô %×,Ò,Ø(à"9ñ
Ðð ×&Ñ&€KØ×ÑÐ,×-Ø�<‰<×!×!Ü×ÑØjôð &+ˆE�L‰LÔ"÷	Eð 	Eö
 õ	Fñ ØØ%Ø-Ø"7ñ	
‰ñ Øð
à-Ø"7ñ
ð ñ	
ˆð 38È÷ ð /€BÔà€Løôi ó hÜÐfÓgÐgðhús    !F ÆF+c           	      ó¼  • SSK Js  Jn  Ub  UOUR                  n[        R                  SU 35        UR                  5         UR                  SS9(       aJ  [        R                  " USS9  [        R                  " U[        R                  R                  US5      5        [        4nUR                  S	5        U(       Ga\  U R!                  5       U R#                  5       S
.n[        R                  R                  USUR$                   SUR&                   S[(         35      n	UR                  X‰SS9  UR                  S5        UR*                  (       aÆ  SSKJn
  U
" [        R                  R                  US5      S[(         3SS9u  p¼U R0                  R0                  n UR3                  U 5      n[5        X×5      (       a  UR7                  X[S9  GOd[        R                  S5        UR                  U[        R                  R                  U[(        5      5        GO[5        X5      (       dÑ  [5        UR3                  U 5      U5      (       aH  UR3                  U 5      R7                  UUR*                  UR9                  U R!                  5       5      S9  O¡[        R                  S5        UR9                  U R!                  5       5      nUR                  U[        R                  R                  U[(        5      5        O8U R7                  UUR*                  UR9                  U R!                  5       5      S9  Ub$  UR*                  (       a  UR7                  U5        ggg)ah  
Saves a model checkpoint on TPU/XLA devices.

Handles FSDP v1 sharded checkpoints (with consolidation on master), as well as
standard XLA model saving via `save_pretrained` or `xm.save`.

Args:
    model (`torch.nn.Module`): The model to save.
    args (`TrainingArguments`): The training arguments.
    accelerator (`Accelerator`): The accelerator instance.
    processing_class: The processing class (tokenizer/processor) to save alongside the model.
    is_fsdp_xla_v1_enabled (`bool`): Whether FSDP XLA v1 is enabled.
    output_dir (`str`, *optional*): The directory to save to. Defaults to `args.output_dir`.
r   NzSaving model checkpoint to F)ÚlocalT)Úexist_okztraining_args.binÚsaving_checkpoint)rd   Úshard_metadataÚrankz-of-Ú-)Úmaster_onlyÚsave_full_checkpoints)Ú%consolidate_sharded_model_checkpointsÚ zrank*-of-*-)Úckpt_prefixÚckpt_suffixÚ
save_model)Ú
state_dictzETrainer.model is not a `PreTrainedModel`, only saving its state dict.)Úis_main_processrz   )rN   rO   rP   Ú
output_dirra   ÚinforF   Úis_master_ordinalÚosÚmakedirsr8   ÚsaveÚpathÚjoinr   Ú
rendezvousrz   Úget_shard_metadataÚprocess_indexÚ
world_sizer   Úshould_saverR   ru   ÚmoduleÚunwrap_modelr   Úsave_pretrainedÚ_maybe_convert_to_cpu)rd   r-   ÚacceleratorÚprocessing_classÚis_fsdp_xla_v1_enabledr|   rK   Úsupported_classesÚckptÚ	ckpt_pathru   Úfull_state_dictÚ_Úunwrapped_modelrz   s                  r   Úsave_tpu_checkpointr–   ¥   s©  € ÷ *Ð)à)Ñ5‘¸4¿?¹?€Jä
‡K�KÐ-¨j¨\Ð:Ô;Ø‡L�L„Nà	×Ñ %ÐÖ(Ü
�Š�J¨Ò.Ü�
Š
�4œŸ™Ÿ™ jÐ2EÓFÔGô (Ð)ÐØ‡M�MÐ%Ô&ßà×%Ñ%Ó'Ø#×6Ñ6Ó8ñ
ˆô —G‘G—L‘L ¨t°D×4FÑ4FÐ3GÀtÈDÏOÉOÐK\Ð\]Ô^jÐ]kÐ-lÓmˆ	à
�‰�¨UˆÑ3à
�‰Ð-Ô.à××ÝXá!FÜŸG™GŸL™L¨°RÓ8Ø)¬,¨Ð8Ø ñ"ÑˆOð
 —L‘L×'Ñ'ˆEØ)×6Ñ6°uÓ=ˆOÜ˜/×=Ñ=Ø×/Ñ/°
Ð/ÓWä—‘ÐcÔdØ—‘˜¬¯©¯©°jÄ,Ó)OÔPùÜ˜×1Ñ1Ü�k×.Ñ.¨uÓ5Ð7H×IÑIØ×$Ñ$ UÓ+×;Ñ;ØØ $× 0Ñ 0Ø×3Ñ3°E×4DÑ4DÓ4FÓGð <ò ô �K‰KÐ_Ô`Ø×1Ñ1°%×2BÑ2BÓ2DÓEˆJØ�G‰G�J¤§¡§¡¨Z¼Ó FÕGà×ÑØØ ×,Ñ,Ø×/Ñ/°×0@Ñ0@Ó0BÓCð 	ñ 	
ð
 Ñ#¨×(8×(8Ø×(Ñ(¨Õ4ð )9Ð#r   r*   )rY   r   r8   Útorch.utils.datar   Úutilsr   r   r   r   Ú
get_loggerÚ__name__ra   r   rk   r–   r+   r   r   Ú<module>r›      sD   ðó Û 	ã Ý 'ç QÓ Qð 
×	Ò	˜HÓ	%€ð Jô ò&tõnJ5r   