ó
    Eñiº  ã            	       ó  • S SK r S SKrS SKJr  S SKJr  S SKJr  S SKJ	r	  S SK
r
S SKJr  S SKJs  Js  Jr  S SKJrJrJr  \ R,                  " \5      r " S S5      rS	\
R4                  R6                  S
\\\\\\   4   4   4S jr g)é    N)Údefaultdict)ÚIterator)Úcontextmanager)ÚEnum)Ú_apply_to_modulesÚ_get_module_fsdp_stateÚclean_tensor_namec                   óÌ   • \ rS rSr%  " S S\\5      r\" \5      r	\
\\4   \S'   \" 5       r\\   \S'   \SS j5       r\\S	\S\S   4S
 j5       5       r\S\SS4S j5       rSrg)ÚSimpleProfileré   c                   ó,   • \ rS rSrSrSrSrSrSrSr	Sr
g	)
ÚSimpleProfiler.Typeé   ÚallÚ
all_gatherÚall_gather_objectÚ
reshardingÚH2DÚD2H© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__ÚALLÚ	ALLGATHERÚALLGATHER_OBJÚ
RESHARDINGr   r   Ú__static_attributes__r   ó    Ú`/home/mande/repo/quber/.venv/lib/python3.13/site-packages/torch/distributed/fsdp/_debug_utils.pyÚTyper      s    † ØˆØ ˆ	Ø+ˆØ!ˆ
ØˆØ‹r    r"   ÚresultsÚ	profilingÚreturnNc                 ól   • U R                   R                  5         U R                  R                  5         g )N)r#   Úclearr$   )Úclss    r!   ÚresetÚSimpleProfiler.reset"   s"   € à�‰×ÑÔØ�‰×ÑÕr    Úprofile_typec              #   óÞ  #   • XR                   ;   a  [        U S35      eU R                   R                  U5        [        R                  " 5       n S v •  [        R                  " 5       nU R
                  U==   X2-
  -  ss'   U R                   R                  U5        g ! [        R                  " 5       nU R
                  U==   X2-
  -  ss'   U R                   R                  U5        f = f7f)Nzk is already being profiled. SimpleProfiler does not support profiling multiple instances at the same time. )r$   ÚAssertionErrorÚaddÚtimeÚ	monotonicr#   Úremove)r(   r+   ÚbeginÚends       r!   ÚprofileÚSimpleProfiler.profile'   sÀ   é € ð Ÿ=™=Ó(Ü Ø�.ð !"ð "óð ð 	�‰×Ñ˜,Ô'Ü—’Ó ˆð	/Ûä—.’.Ó"ˆCØ�K‰K˜Ó%¨©Ñ4Ó%Ø�M‰M× Ñ  Õ.øô —.’.Ó"ˆCØ�K‰K˜Ó%¨©Ñ4Ó%Ø�M‰M× Ñ  Õ.üs    ‚AC-ÁB ÁA
C-ÂAC*Ã*C-Úmsgc                 óø   • [         R                  " 5       S:X  aQ  [         R                  " 5       [         R                  R                  :X  a   [
        R                  SXR                  5        U R                  5         g )Nr   z%s %s)	ÚdistÚget_rankÚget_debug_levelÚ
DebugLevelÚINFOÚloggerÚinfor#   r)   )r(   r6   s     r!   Údump_and_resetÚSimpleProfiler.dump_and_reset:   sG   € ô �=Š=‹?˜aÓ¤D×$8Ò$8Ó$:¼d¿o¹o×>RÑ>RÓ$RÜ�K‰K˜ §k¡kÔ2Ø�	‰	�r    r   )r%   N)r   r   r   r   Ústrr   r"   r   Úfloatr#   ÚdictÚ__annotations__Úsetr$   Úclassmethodr)   r   r   r4   r?   r   r   r    r!   r   r      sœ   ‡ ôˆs�Dô ñ !,¨EÓ 2€GˆT�#�u�*ÑÓ2Ù›%€Iˆs�3‰xÓàóó ðð Øð/ 3ð /¨8°D©>ó /ó ó ð/ð" ð ð ¨ó ó ór    r   Úmodelr%   c                 óŠ   • S nS nS/n0 n[        U UUU R                  5        VVs/ s H  u  pVUPM	     snnUU5      $ s  snnf )ao  
It is used for composable fully_shard() code path, it returns
  1. sharded module tree info: each line represents a submodule name that contains the
submodule's FQN and its submodule class name, if the submodule is sharded by `fully_shard`,
the submodule name will add a postfix with ' FULLY SHARDED'. Each increased tree
level adds 4 spaces before the printed name. A printed sharded module tree info for a toy model
is like this:
    [CompositeModel] FULLY SHARDED
        l1[Linear]
        u1[UnitModule] FULLY SHARDED
            u1.l1[Linear]
            u1.seq[Sequential]
                u1.seq.0[ReLU]
                u1.seq.1[Linear]
                u1.seq.2[ReLU]
            u1.l2[Linear]
        u2[UnitModule] FULLY SHARDED
            u2.l1[Linear]
            u2.seq[Sequential]
                u2.seq.0[ReLU]
                u2.seq.1[Linear]
                u2.seq.2[ReLU]
            u2.l2[Linear]
        l2[Linear]
  2. a dict mapping from the concated module FQN and class name to a list of its managed
original parameters' FQNs. An example of the dict for the above toy sharded model is like this:
        {'[CompositeModel]': ['l1.weight', 'l1.bias', 'l2.weight', 'l2.bias'],
         'u1[UnitModule]': ['u1.l1.weight', 'u1.l1.bias', 'u1.seq.1.weight', 'u1.seq.1.bias', 'u1.l2.weight', 'u1.l2.bias'],
         'u2[UnitModule]': ['u2.l1.weight', 'u2.l1.bias', 'u2.seq.1.weight', 'u2.seq.1.bias', 'u2.l2.weight', 'u2.l2.bias']
        }
All FQNs are prefixed starting from ``model``.

Args:
    model (torch.nn.Module): Root module (which may or may not be passed to
                             composable `fully_shard()`).
c                 óŒ  • US-  n[        U5      S:”  a  US   S:X  a  US S OUnUS-   U R                  R                  -   S-   nSU-  U-   n[        U 5      n	U	c  US==   US-   -  ss'   g U	R                  R                  U S 5      n
U
(       a  US==   US	-   S-   -  ss'   OUS==   US-   -  ss'   U
(       a‡  U
R                  n[        U[        R                  5      (       d  [        S
[        U5       35      eUR                   Vs/ s H  n[        X-   5      PM     nnXt;   a  XG   R                  U5        g XÔU'   g g s  snf )Né   r   éÿÿÿÿÚ.Ú[Ú]Ú Ú
z FULLY SHARDEDzExpected FlatParameter, got )ÚlenÚ	__class__r   r   Ú_fully_sharded_module_to_handleÚgetÚ
flat_paramÚ
isinstanceÚflat_param_fileÚFlatParameterr-   ÚtypeÚ_fqnsr	   Úextend)ÚmoduleÚprefixÚ
tree_levelÚsharded_tree_infoÚsharded_module_name_to_fqnsÚ
num_spacesÚtrimed_prefixÚprefixed_module_nameÚprinted_prefixed_module_nameÚstateÚhandleÚparamÚnameÚglobal_fqnss                 r!   Ú	module_fnÚD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.module_fnk   sl  € ð   !‘^ˆ
ä ›K¨!›O°°r±
¸cÓ0AˆF�3�B‰KÈð 	ð  -¨sÑ2°V×5EÑ5E×5NÑ5NÑNÐQTÑTÐØ'*¨ZÑ'7Ð:NÑ'NÐ$ä& vÓ.ˆØ‰=Ø˜aÓ Ð$@À4Ñ$GÑGÓ Øà×6Ñ6×:Ñ:¸6À4ÓHˆæØ˜aÓ Ø,Ð/?Ñ?À$ÑFñÔ ð ˜aÓ Ð$@À4Ñ$GÑGÓ æØ×%Ñ%ˆEÜ˜e¤_×%BÑ%B×CÑCÜ$Ð'CÄDÈÃKÀ=Ð%QÓRÐRà=B¿[º[óÚ=H°TÔ! &¡-Ö0¹[ð ð ð $ÓBØ+ÑA×HÑHÈÕUàDOÐ,@ÒAð ùòs   Ä	Ec                 ó   • U S   U4$ )Nr   r   )r_   r`   s     r!   Ú	return_fnÚD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.return_fn�   s   € Ø  Ñ#Ð%@Ð@Ð@r    Ú )r   Únamed_parameters)rG   rj   rm   r_   r`   ÚkeyÚ_s          r!   Ú1_get_sharded_module_tree_with_module_name_to_fqnsrs   C   se   € òP#PòJAð
 	ð$Ðð 9;ÐÜØØØØ ×1Ñ1Ô3Ô4Ò3‘�‹Ñ3Ò4ØØ#óð ùó 	5s   §?)!Úloggingr/   Úcollectionsr   Úcollections.abcr   Ú
contextlibr   Úenumr   ÚtorchÚtorch.distributedÚdistributedr8   Ú"torch.distributed.fsdp._flat_paramÚfsdpÚ_flat_paramrW   Ú$torch.distributed.fsdp._common_utilsr   r   r	   Ú	getLoggerr   r=   r   ÚnnÚModuleÚtuplerA   rC   Úlistrs   r   r    r!   Ú<module>r…      s�   ðã Û Ý #Ý $Ý %Ý ã Ý  ß <Ó <÷ñ ð 
×	Ò	˜8Ó	$€÷*ñ *ðZ\Ø�8‰8�?‰?ð\à
ˆ3��S˜$˜s™)�^Ñ$Ð$Ñ%õ\r    