ó
    >:j.  ã                  ó²   • S SK Jr  S SKrS SKJr  S SKrS SKJr  S SK	J
r
  S SKJ
s  Jr  S SKJr  S SKJrJrJr  S SKJrJr   " S S\
R.                  \5      rg)	é    )ÚannotationsN)ÚOptional)Ú
BufferDict)ÚBaseTunerLayerÚ_get_in_out_featuresÚcheck_adapters_to_merge)Úcheck_deepspeed_zero3_enabledÚgather_params_ctxc                  ó¨   ^ • \ rS rSrSrSr S         SU 4S jjjr\S 5       rSS jr	S r
S rSSS	 jjrSS
 jrSS jrSS jrSS jrSrU =r$ )ÚTrainableTokensLayeré   )Útrainable_tokens_delta)Útoken_indicesÚtrainable_tokens_originalc                óî  >• [         TU ]  5         Xl        X l        XPl        U(       a  U/O/ U l        U R                  (       d3  [        R                  " 0 5      U l	        [        0 5      U l        0 U l        OQU R                  R                  U l	        U R                  R                  U l        U R                  R                  U l        / U l        [        U R                  5       5      u  pgX`l        Xpl        g ©N)ÚsuperÚ__init__Ú
base_layerÚ_active_adapterÚkwargsÚ_tied_adapterÚtied_adapterÚnnÚParameterDictr   r   r   r   Úmerged_adaptersr   Úget_base_layerÚin_featuresÚout_features)	Úselfr   Úadapter_namer   r   r   r   r   Ú	__class__s	           €Ú_/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/trainable_tokens/layer.pyr   ÚTrainableTokensLayer.__init__%   sÊ   ø€ ô 	‰ÑÔà$ŒØ+ÔØŒö 0<˜l™^ÀˆÔð × × Ü*,×*:Ò*:¸2Ó*>ˆDÔ'Ü-7¸«^ˆDÔ*Ø!#ˆDÕà*.×*;Ñ*;×*RÑ*RˆDÔ'Ø-1×->Ñ->×-XÑ-XˆDÔ*Ø!%×!2Ñ!2×!@Ñ!@ˆDÔð  "ˆÔä$8¸×9LÑ9LÓ9NÓ$OÑ!ˆØ&ÔØ(Õó    c                óD   • U R                   (       a  U R                   S   $ g )Nr   )r   )r    s    r#   r   Ú!TrainableTokensLayer.tied_adapterL   s   € à××Ø×%Ñ% aÑ(Ð(Ør%   c                óú  • Sn[         R                  " S[         R                  R                  5       5      n[	        U/SS9   [
        R                  " 5       (       aF  [
        R                  " 5       (       a,  [
        R                  " 5       U:X  a  X   R                  5       nO+[         R                  " [        U5      U4UR                  US9nSSS5        [
        R                  " WUS9  U$ ! , (       d  f       N%= f)z´DeepSpeed zero3 specific code to initialize trainable tokens.

Ensures that only the necessary weights are collected to a single rank, initialized, and then shared with all
ranks.
r   ÚcudaN)Úmodifier_rank©ÚdtypeÚdevice)Úsrc)Útorchr-   r)   Úcurrent_devicer
   ÚdistÚis_availableÚis_initializedÚget_rankÚcloneÚemptyÚlenr,   Ú	broadcast)r    ÚweightÚrowsÚ	embed_dimÚsrc_rankr-   Útoken_weightss          r#   Ú_collect_token_weightsÚ+TrainableTokensLayer._collect_token_weightsR   s·   € ð ˆä—’˜f¤e§j¡j×&?Ñ&?Ó&AÓBˆä ˜x°tÓ<Ü× Ò ×"Ñ"¤t×':Ò':×'<Ñ'<ÄÇÂÃÐT\ÓA\Ø &¡× 2Ñ 2Ó 4‘ô !&§¢Ü˜“Y 	Ð*Ø Ÿ,™,Ø!ñ!�÷ =ô 	�Š�}¨(Ò3ØÐ÷ =Õ<ús   ÁBC,Ã,
C:c                ó>  • UR                  SS 5      (       a  g US   U R                  U'   UR                  SS5      nU R                  5       R                  n[	        U R                  5       S5      (       a  U R                  5       R
                  nOU R                  5       R                  nU(       aL  [        5       (       a   U R                  X@R                  U   U5      nO_U R                  U R                  U      nOB[        R                  " [        U R                  U   5      U4UR                  UR                  S9n[        R                  " UR!                  5       SS9U R"                  U'   UR!                  5       U R$                  U'   U R'                  U5        g )Nr   r   Úinit_weightsTÚembedding_dimr+   )Úrequires_grad)Úgetr   r   r9   ÚhasattrrB   r   r	   r>   r/   Úrandnr7   r,   r-   r   Ú	Parameterr5   r   r   Ú%_move_adapter_to_device_of_base_layer)r    r!   r   rA   r9   r;   Úvaluess          r#   Úupdate_layerÚ!TrainableTokensLayer.update_layerk   sS  € Ø�:‰:�n d×+Ñ+àà+1°/Ñ+Bˆ×Ñ˜<Ñ(Ø—z‘z .°$Ó7ˆð ×$Ñ$Ó&×-Ñ-ˆä�4×&Ñ&Ó(¨/×:Ñ:Ø×+Ñ+Ó-×;Ñ;‰Ið ×+Ñ+Ó-×9Ñ9ˆIæÜ,×.Ñ.Ø×4Ñ4°V×=OÑ=OÐP\Ñ=]Ð_hÓi‘àŸ™ T×%7Ñ%7¸Ñ%EÑF‘ô —[’[Ü�T×'Ñ'¨Ñ5Ó6¸	ÐBØ—l‘lØ—}‘}ñˆFô 57·L²LÀÇÁÃÐ_cÑ4dˆ×#Ñ# LÑ1Ø7=·|±|³~ˆ×&Ñ& |Ñ4à×2Ñ2°<Õ@r%   c                ó"  • [        U5      S::  a  g[        5       n[        XR                  -   5       HZ  n[        U R                  U   5      n[        UR	                  U5      5      (       a  [        SU S35      eUR                  U5        M\     g)zìRaises an error if the token indices of the given adapter names are overlapping.
This is currently not supported and can lead to undefined behavior of the model if no specific merging between
the overlapping indices' values is applied.
é   NzToken indices of adapter zy are already defined and would result in undefined merging behavior. Only disjunct token indices are currently supported.)r7   Úsetr   r   ÚintersectionÚ
ValueErrorÚupdate)r    Úadapter_namesÚindicesr!   Ú	index_sets        r#   Ú_check_overlapping_tokensÚ.TrainableTokensLayer._check_overlapping_tokens”   s�   € ô
 ˆ}Ó Ó"Øä“%ˆô   ×0DÑ0DÑ DÖEˆLÜ˜D×.Ñ.¨|Ñ<Ó=ˆIÜ�7×'Ñ'¨	Ó2×3Ñ3Ü Ø/°¨~ð >gð góð ð �N‰N˜9Ö%ò Fr%   c                ó^  • [        X5      nU(       d  g U R                  U5        U R                  R                  R                  nU H®  n[
        R                  " U R                  U   5      R                  UR                  5      nU R                  U   R                  U5      nUR                  SXVS9nU(       d  Mv  [
        R                  " U5      R                  5       (       a  M¡  [        SU S35      e   X0R                  R                  l        U R                  R!                  U5        g )Nr   ©ÚdimÚindexÚsourcez1NaNs detected in the merged weights. The adapter z seems to be broken)r   rU   r   r9   Údatar/   Útensorr   Útor-   r   Ú
index_copyÚisfiniteÚallrP   r   Úextend)r    Ú
safe_mergerR   Úmergedr!   rZ   Údeltass          r#   ÚmergeÚTrainableTokensLayer.merge¨   sñ   € Ü/°ÓDˆæàà×&Ñ& }Ô5à—‘×'Ñ'×,Ñ,ˆã)ˆLÜ—L’L ×!3Ñ!3°LÑ!AÓB×EÑEÀfÇmÁmÓTˆEØ×0Ñ0°Ñ>×AÑAÀ&ÓIˆFØ×&Ñ&¨1°EÐ&ÐIˆFçˆz¤%§.¢.°Ó"8×"<Ñ"<×">Ó">Ü Ð#TÐUaÐTbÐbuÐ!vÓwÐwñ *ð '-�‰×ÑÔ#Ø×Ñ×#Ñ# MÕ2r%   c                óR  • U R                   (       d  [        R                  " S5        g [        U R                  5      S:”  aæ  U R                  R                  5       n[        R                  " U R                  U   5      R                  U R                  R                  R                  5      nU R                  U   R                  U R                  R                  5      nU R                  R                  R                  R                  SX#S9  [        U R                  5      S:”  a  Må  g g )Nz Already unmerged. Nothing to do.r   rX   )rd   ÚwarningsÚwarnr7   r   Úpopr/   r]   r   r^   r   r9   r-   r   r\   Úindex_copy_)r    r!   rZ   Ú	originalss       r#   ÚunmergeÚTrainableTokensLayer.unmerge¾   sÑ   € Ø�{�{Ü�MŠMÐ<Ô=Øä�$×&Ñ&Ó'¨!Ó+Ø×/Ñ/×3Ñ3Ó5ˆLä—L’L ×!3Ñ!3°LÑ!AÓB×EÑEÀdÇoÁo×F\ÑF\×FcÑFcÓdˆEØ×6Ñ6°|ÑD×GÑGÈÏÉ×H^ÑH^Ó_ˆIØ�O‰O×"Ñ"×'Ñ'×3Ñ3¸ÀÐ3ÑYô �$×&Ñ&Ó'¨!×+r%   c                ó  • U R                   R                  nU Hm  n[        R                  " U R                  U   5      R                  UR                  5      nU R                  U   R                  U5      nUR                  SXES9nMo     U$ )Nr   rX   )	r   r9   r/   r]   r   r^   r-   r   r_   )r    Úactive_adaptersÚWr!   rZ   re   s         r#   Úget_merged_weightsÚ'TrainableTokensLayer.get_merged_weightsÊ   sw   € Ø�O‰O×"Ñ"ˆã+ˆLÜ—L’L ×!3Ñ!3°LÑ!AÓB×EÑEÀaÇhÁhÓOˆEØ×0Ñ0°Ñ>×AÑAÀ!ÓDˆFØ—‘ ¨%�Ð?ŠAñ ,ð ˆr%   c           
     óª  • U R                   (       d  U(       d9  U R                  (       a  U R                  5         U R                  " U/UQ70 UD6nU$ U R                  (       a  U R                  " U/UQ70 UD6nU$ U R	                  U5        U R                  U5      n[        U R                  [        R                  R                  5      (       a°  [        R                  " UUU R                  R                  U R                  R                  U R                  R                  U R                  R                  U R                  R                   S9nU R#                  5       nUb  XWR%                  UR&                  5      -  nU$ [        U R                  [        R                  R(                  5      (       a  [        R*                  " UUS9nU$ [-        S5      e)N)Úinputr9   Úpadding_idxÚmax_normÚ	norm_typeÚscale_grad_by_freqÚsparse)rv   r9   zZTrainableTokensLayer wraps an unknown layer type, maybe you are targeting the wrong layer?)Údisable_adaptersrd   rn   r   rU   rs   Ú
isinstancer/   r   Ú	EmbeddingÚFÚ	embeddingrw   rx   ry   rz   r{   Ú_get_embed_scaler^   r,   ÚLinearÚlinearrP   )r    Úxrq   Úargsr   Úresultrr   Úembed_scales           r#   Úforward_adaptersÚ%TrainableTokensLayer.forward_adapters×   st  € Ø× × ®Ø�{�{Ø—‘”Ø—_’_ QÐ8¨Ò8°Ñ8ˆFðP ˆðO �[�[Ø—_’_ QÐ8¨Ò8°Ñ8ˆFðL ˆðI ×*Ñ*¨?Ô;à×'Ñ'¨Ó8ˆAô ˜$Ÿ/™/¬5¯8©8×+=Ñ+=×>Ñ>ÜŸšØØØ $§¡× ;Ñ ;Ø!Ÿ_™_×5Ñ5Ø"Ÿo™o×7Ñ7Ø'+§¡×'IÑ'IØŸ?™?×1Ñ1ñ�ð #×3Ñ3Ó5�ØÑ*Ø#§n¡n°V·\±\Ó&BÑB�Fð ˆô ˜DŸO™O¬U¯X©X¯_©_×=Ñ=äŸšØØñ�ð ˆô	 !Øpóð r%   c                óB   • U R                   " XR                  /UQ70 UD6$ r   )rˆ   rq   )r    r„   r…   r   s       r#   ÚforwardÚTrainableTokensLayer.forward  s#   € Ø×$Ò$ Q×(<Ñ(<ÐN¸tÒNÀvÑNÐNr%   )
r   r   r   r   r   r   r   r   r   r   r   )
r   z	nn.Moduler!   Ústrr   z	list[int]r   zOptional[TrainableTokensLayer]ÚreturnÚNone)r9   útorch.Tensorr:   r�   r;   ÚintrŽ   r�   )FN)rc   ÚboolrR   zOptional[list[str]]rŽ   r�   )rŽ   r�   )rŽ   r�   )r„   r�   rŽ   r�   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Úadapter_layer_namesÚother_param_namesr   Úpropertyr   r>   rJ   rU   rf   rn   rs   rˆ   r‹   Ú__static_attributes__Ú__classcell__)r"   s   @r#   r   r      s™   ø† à5Ðð GÐð 8<ð%)àð%)ð ð%)ð !ð	%)ð
 5ð%)ð 
÷%)ð %)ðN ñó ðô
ò2'AòR&ö(3ô,
Zôô,÷\Oò Or%   r   )Ú
__future__r   ri   Útypingr   r/   Útorch.distributedÚdistributedr1   Útorch.nnr   Útorch.nn.functionalÚ
functionalr   Úpeft.tuners._buffer_dictr   Úpeft.tuners.tuners_utilsr   r   r   Úpeft.utils.integrationsr	   r
   ÚModuler   © r%   r#   Ú<module>r¨      sA   ðõ #ã Ý ã Ý  Ý ß Ð å /ß bÑ bß TôhO˜2Ÿ9™9 nõ hOr%   