ó
    Eñi{  ã                   ó°  • % S SK JrJr  S SKJr  S SKrS SKJr  S SK	J
r
  S SKJr  S SKJrJrJr  S SKJrJr   " S S	\5      rSq\S-  \S
'   S\SS4S jr SS\R0                  S\S-  S\\R0                  \S-  4   4S jjr SS\R0                  S\S\S-  S\R0                  4S jjr SS\R0                  S\S\S\S\R:                  S\S-  S\R0                  4S jjr SS\R0                  S\S\S\S-  S\R0                  4
S jjr SS\R0                  S\S-  S\\R0                  \ \   4   4S jjr! SS\S\S-  S\S-  S\R0                  4S jjr"g)é    )ÚABCÚabstractmethod)ÚAnyN)ÚShardedTensor)ÚShard)Ú_all_gather_dtensorÚ_create_chunk_dtensorÚ_create_chunk_sharded_tensor)Ú
DeviceMeshÚDTensorc                   ó  • \ rS rSrSr\S\R                  S\\R                  \	S-  4   4S j5       r
\S\R                  S\	S\R                  4S j5       r\ SS\R                  S	\S
\S\S\R                  S\R                  S-  S\R                  4S jj5       r\S\R                  S	\S\S\R                  4S j5       r\S\R                  S\\R                  \\   4   4S j5       r\S\S\S-  S\R                  4S j5       rSrg)ÚFSDPExtensionsé   zÔ
This enables some customizable hooks to enable composability with tensor
parallelism. To activate these hooks, use :func:`_set_fsdp_extensions` to
set a custom :class:`FSDPExtensions` that implements the hooks.
ÚtensorÚreturnNc                 ó   • g)z6E.g. converting ``DistributedTensor`` to local tensor.N© ©Úselfr   s     Úd/home/mande/repo/quber/.venv/lib/python3.13/site-packages/torch/distributed/fsdp/_fsdp_extensions.pyÚpre_flatten_transformÚ$FSDPExtensions.pre_flatten_transform   s   € ð 	ó    Úparam_extensionc                 ó   • g)z6E.g. converting local tensor to ``DistributedTensor``.Nr   )r   r   r   s      r   Úpost_unflatten_transformÚ'FSDPExtensions.post_unflatten_transform   s   € ð 	r   ÚrankÚ
world_sizeÚnum_devices_per_nodeÚpgÚdevicec                 ó   • g)z6Shards a tensor to chunks and returns the local chunk.Nr   )r   r   r   r   r    r!   r"   s          r   Úchunk_tensorÚFSDPExtensions.chunk_tensor(   ó   € ð 	r   Údevice_meshc                 ó   • g)zAShards a tensor/DTensor to DTensor and returns the local DTensor.Nr   )r   r   r   r'   s       r   Úchunk_dtensorÚFSDPExtensions.chunk_dtensor5   s   € ð 	r   c                 ó   • g)z‹
This is to be called before loading a *sharded* model state dict and
should return the tensor and list of shards from which to load data.
Nr   r   s     r   Úpre_load_state_dict_transformÚ,FSDPExtensions.pre_load_state_dict_transform?   s   € ð 	r   Úparent_meshc                 ó   • g)z“
This is to be called before loading a *sharded* DTensor state dict.
This gathers tensor in FSDP dimension and returns local tensor of
TP DTensor.
Nr   )r   r   r.   s      r   Úall_gather_dtensorÚ!FSDPExtensions.all_gather_dtensorJ   r&   r   r   ©N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   ÚtorchÚTensorÚtupler   r   r   ÚintÚdistÚProcessGroupr"   r$   r   r)   Úlistr   r,   r   r0   Ú__static_attributes__r   r   r   r   r      s¦  † ñð ðà—‘ðð 
ˆu�|‰|˜S 4™ZÐ'Ñ	(óó ðð ðà—‘ðð ðð 
�‰ó	ó ðð ð '+ñ
à—‘ð
ð ð
ð ð	
ð
 "ð
ð ×Ñð
ð —‘˜tÑ#ð
ð 
�‰ô
ó ð
ð ðà—‘ðð ðð  ð	ð
 
�‰óó ðð ðà—‘ðð 
ˆu�|‰|˜T %™[Ð(Ñ	)óó ðð ð
àð
ð   $Ñ&ð
ð 
�‰ó	
ó ó
r   r   Ú_extensionsÚ	flattenerr   c                 ó   • U q g r2   )r@   )rA   s    r   Ú_set_fsdp_extensionsrC   [   s   € à�Kr   r   Úfsdp_extensionc                 óB   • Ub  UR                  U 5      u  p#Ub  X#4$ U S 4$ r2   )r   )r   rD   Ú
new_tensorr   s       r   Ú_ext_pre_flatten_transformrG   `   s6   € ð Ñ!Ø&4×&JÑ&JÈ6Ó&RÑ#ˆ
ØÑ&ØÐ.Ð.Ø�4ˆ<Ðr   r   c                 ó4   • Ub  Ub  UR                  X5      $ U $ r2   )r   )r   r   rD   s      r   Ú_ext_post_unflatten_transformrI   k   s%   € ð
 Ñ! oÑ&AØ×6Ñ6°vÓOÐOØ€Mr   r   r   r    r!   c                 óD   • Ub  UR                   O[        nU" U UUUU5      $ r2   )r$   r
   )r   r   r   r    r!   rD   Úchunk_tensor_fns          r   Ú_ext_chunk_tensorrL   u   s;   € ð Ñ%ð 	×#Ò#ä)ð ñ
 ØØØØØ
óð r   r'   c                 ó@   • Ub  UR                   O[        nU" U UU5      $ r2   )r)   r	   )r   r   r'   rD   Úchunk_dtensor_fns        r   Ú_ext_chunk_dtensorrO   ‹   s5   € ð Ñ%ð 	×$Ò$ä"ð ñ
 ØØØóð r   c                 ó¢   • Ub  UR                  U 5      $ [        U 5      [        La  [        S[        U 5       35      eU R	                  5       nX4$ )NzExpected ShardedTensor, got )r,   Útyper   ÚAssertionErrorÚlocal_shards)r   rD   Úshardss      r   Ú"_ext_pre_load_state_dict_transformrU   �   sU   € ð Ñ!Ø×;Ñ;¸FÓCÐCäˆFƒ|œ=Ò(ÜÐ;¼DÀ»L¸>ÐJÓKÐKØ× Ñ Ó"€FØÐÐr   r.   c                 ó<   • Ub  UR                   O[        nU" X5      $ r2   )r0   r   )r   r.   rD   Úall_gather_dtensor_fns       r   Ú_ext_all_gather_dtensorrX   ª   s*   € ð Ñ%ð 	×)Ò)ä ð ñ
 ! Ó5Ð5r   r2   )#Úabcr   r   Útypingr   r8   Útorch.distributedÚdistributedr<   Ú+torch.distributed._shard.sharded_tensor.apir   Ú-torch.distributed._shard.sharded_tensor.shardr   Ú#torch.distributed.fsdp._shard_utilsr   r	   r
   Útorch.distributed.tensorr   r   r   r@   Ú__annotations__rC   r9   r:   rG   rI   r;   r=   rL   rO   r>   rU   rX   r   r   r   Ú<module>rb      sõ  ðß #Ð #Ý ã Ý  Ý EÝ ?÷ñ ÷
 9ôE�Sô EðP &*€ˆ^˜dÑ"Ó )ð Nð °tô ð -1ñØ�L‰Lðà" TÑ)ðð ˆ5�<‰<˜˜t™Ð#Ñ$õð -1ñØ�L‰Lðàðð # TÑ)ðð ‡\�\õ	ð  -1ñØ�L‰Lðà
ðð ðð ð	ð
 	×Ñðð # TÑ)ðð ‡\�\õð4 -1ñ	Ø�L‰Lðà
ðð ðð # TÑ)ð	ð
 ‡\�\õð( -1ñ
Ø�L‰Lð
à" TÑ)ð
ð ˆ5�<‰<˜˜e™Ð$Ñ%õ
ð  -1ñ
6Øð
6à˜dÑ"ð
6ð # TÑ)ð
6ð ‡\�\ö	
6r   