ó
    >:j  ã                  ól   • S r SSKJr  SSKJr  SSKrSSKJr  SSKJ	r	  / SQr
S rSS jrSS	 jrSS
 jrg)z=Utilities for Orthogonal Subspace Learning with Adaptive OSF.é    )Úannotations)ÚAnyN)Únn)Údecompose_weight_matrixÚ$project_gradient_to_orthogonal_spaceÚreconstruct_weight_matrixc                óH   • [        U S5      (       a  U R                  5       $ U $ )zHWait for AsyncCollectiveTensor if needed, otherwise return tensor as-is.Úwait)Úhasattrr
   )Útensors    ÚR/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/osf/utils.pyÚ_wait_if_asyncr   #   s    € äˆv�v×ÑØ�{‰{‹}ÐØ€Mó    c                ó®  • U R                   nU R                  nU R                  [        R                  5      n[        R
                  R                  USS9u  pVn[        XR                  S   5      nUSS2SU24   R                  5       R                  5       R                  X#S9USU R                  5       R                  5       R                  X#S9USU2SS24   R                  5       R                  5       R                  X#S9[        R                  " USS2US24   R                  5       R                  5       R                  X#S95      [        R                  " XhS R                  5       R                  5       R                  X#S95      [        R                  " XxS2SS24   R                  5       R                  5       R                  X#S95      US.n	U	$ )zJPerform an SVD of ``weight`` and split it into frozen and trainable parts.F)Úfull_matricesr   N)ÚdeviceÚdtype)ÚU_highÚS_highÚV_highÚU_lowÚS_lowÚV_lowÚ	rank_high)r   r   ÚtoÚtorchÚfloat32ÚlinalgÚsvdÚminÚshapeÚ
contiguousÚdetachr   Ú	Parameter)
ÚweightÚtop_kÚdevice_localÚ
orig_dtypeÚWÚUÚSÚVtÚkr   s
             r   r   r   *   s‰  € à—=‘=€LØ—‘€JØ�	‰	”%—-‘-Ó €AÜ�|‰|×Ñ °ÐÐ7�H€Aˆ"ÜˆE—7‘7˜1‘:Ó€Að ’A�r˜�r�E‘(×%Ñ%Ó'×.Ñ.Ó0×3Ñ3¸<Ð3ÐZØ�B�Q�%×"Ñ"Ó$×+Ñ+Ó-×0Ñ0¸Ð0ÐWØ�R�a�Rš�U‘)×&Ñ&Ó(×/Ñ/Ó1×4Ñ4¸LÐ4Ð[Ü—’˜a¢ 1¡2 ™h×1Ñ1Ó3×:Ñ:Ó<×?Ñ?À|Ð?ÐfÓgÜ—’˜a ˜e×.Ñ.Ó0×7Ñ7Ó9×<Ñ<ÀLÐ<ÐcÓdÜ—’˜b¡¢Q ™i×2Ñ2Ó4×;Ñ;Ó=×@Ñ@ÈÐ@ÐgÓhØñ€Cð €Jr   c                ó~  • U S   nU S   nU S   nU S   nU S   nU S   nUR                  5       S:”  a<  UR                  5       S:”  a(  [        R                  " XR                  S5      -  U5      O=[        R                  " UR                  S5      UR                  S5      UR                  S	9nUR                  5       S:”  a<  UR                  5       S:”  a(  [        R                  " XER                  S5      -  U5      O=[        R                  " UR                  S5      UR                  S5      UR                  S	9nXx-   $ )
z4Reconstruct a weight matrix from its SVD components.r   r   r   r   r   r   r   é   )r   )Únumelr   ÚmmÚ	unsqueezeÚzerosÚsizer   )	Úsvd_dictr   r   r   r   r   r   Ú	high_partÚlow_parts	            r   r   r   >   s  € à�hÑ€FØ�hÑ€FØ�hÑ€FØ�WÑ€EØ�WÑ€EØ�WÑ€Eð �<‰<‹>˜AÓ &§,¡,£.°1Ó"4ô 	�Š�×*Ñ*¨1Ó-Ñ-¨vÔ6ä�[Š[˜Ÿ™ A›¨¯
©
°1«¸f¿m¹mÑLð ð �;‰;‹=˜1Ó §¡£°Ó!2ô 	�Š�Ÿ™¨Ó+Ñ+¨UÔ3ä�[Š[˜Ÿ™ Q›¨¯©°Q«ÀÇÁÑMð ð
 ÑÐr   c                óœ  ^^	^
^• U S   R                   c!  U S   R                   c  U S   R                   c  gU S   mU S   m	U S   R                   Gb*  U S   R                   m
[        TSU4S j5      " 5       n[        T
SU
4S	 j5      " 5       n[        R                  " UR	                  S
S5      U5      n[
        R                  " 5       (       a`  [
        R                  " 5       (       aF  [
        R                  " 5       S:”  a-  [
        R                  " U[
        R                  R                  S9  UR                  XSS9  [        T
S5      (       a  T
R                  R                  U5        OT
R                  U5        U S   R                   GbA  U S   R                   m[        T	SU	4S j5      " 5       n[        TSU4S j5      " 5       n[        R                  " UR	                  S
S5      U5      n[
        R                  " 5       (       a`  [
        R                  " 5       (       aF  [
        R                  " 5       S:”  a-  [
        R                  " U[
        R                  R                  S9  [        R                  " XV5      nUR!                  USS9  [        TS5      (       a  TR                  R                  U5        gTR                  U5        gg)zUProject gradients of ``U_low`` and ``V_low`` to be orthogonal to the high rank space.r   Nr   r   r   r   Úto_localc                 ó   >• T $ ©N© )r   s   €r   Ú<lambda>Ú6project_gradient_to_orthogonal_space.<locals>.<lambda>`   ó   ø€ ¹6r   c                 ó   >• T $ r;   r<   )ÚdUs   €r   r=   r>   a   ó   ø€ ±2r   r   r/   )Úopg      ð¿)ÚalphaÚ_local_tensorc                 ó   >• T $ r;   r<   )r   s   €r   r=   r>   u   r?   r   c                 ó   >• T $ r;   r<   )ÚdVs   €r   r=   r>   v   rB   r   )ÚgradÚgetattrr   r1   Ú	transposeÚdistÚis_availableÚis_initializedÚget_world_sizeÚ
all_reduceÚReduceOpÚSUMÚaddmm_r   rE   Úcopy_Úadd_)r5   Úlocal_U_highÚlocal_dUÚ
proj_coeffÚlocal_V_highÚlocal_dVÚG_localÚupdater   r   rA   rH   s           @@@@r   r   r   T   s/  û€ à�Ñ×ÑÑ%¨(°7Ñ*;×*@Ñ*@Ñ*HÈXÐV]ÑM^×McÑMcÑMkØà�hÑ€FØ�hÑ€Fð �Ñ×ÑÒ)Ø�gÑ×#Ñ#ˆä˜v z´>ÔBÓDˆÜ˜2˜z¬:Ô6Ó8ˆô —X’X˜l×4Ñ4°Q¸Ó:¸HÓEˆ
Ü×Ò×Ñ¤4×#6Ò#6×#8Ñ#8¼T×=PÒ=PÓ=RÐUVÓ=VÜ�OŠO˜J¬4¯=©=×+<Ñ+<Ò=à�‰˜¸ˆÑ=ä�2�×'Ñ'Ø×Ñ×"Ñ" 8Õ,à�H‰H�XÔð �Ñ×ÑÒ)Ø�gÑ×#Ñ#ˆÜ˜v z´>ÔBÓDˆÜ˜2˜z¬:Ô6Ó8ˆô —(’(˜<×1Ñ1°!°QÓ7¸ÓFˆÜ×Ò×Ñ¤4×#6Ò#6×#8Ñ#8¼T×=PÒ=PÓ=RÐUVÓ=VÜ�OŠO˜G¬¯©×(9Ñ(9Ò:ô —’˜(Ó,ˆØ�‰�f DˆÑ)ä�2�×'Ñ'Ø×Ñ×"Ñ" 8Õ,à�H‰H�XÕð% *r   )r%   útorch.Tensorr&   ÚintÚreturnúdict[str, Any])r5   zdict[str, torch.Tensor]r_   r]   )r5   r`   r_   ÚNone)Ú__doc__Ú
__future__r   Útypingr   r   Útorch.distributedÚdistributedrL   r   Ú__all__r   r   r   r   r<   r   r   Ú<module>rh      s5   ðñ Då "å ã Ý  Ý ò€òôô( õ,1r   