ó
    >:jn  ã                   óv   • S SK Jr  S SKJrJr  SSKJrJr  SSKJ	r	J
r
  SSKJr   " S S\R                  5      rg)	é    N)Ú_freeze_adapterÚ_get_submodulesé   )ÚAdaptionPromptConfigÚprepare_config)ÚAdaptedAttentionÚAdaptedAttentionGPT)Úis_adaption_prompt_trainablec                   ó  ^ • \ rS rSrSrS\S\4U 4S jjrS\S\SS4S	 jr	SS\S
\
SS4S jjrS rS rS\S\\R                      SS4S jrS\SS4S jrS\SS4S jrS\R                   SS4S jrS\4U 4S jjrSrU =r$ )ÚAdaptionPromptModelé   a)  
Implements adaption prompts as described in https://huggingface.co/papers/2303.16199.

The top L attention modules are replaced with AdaptedAttention modules that wrap the original ones, but insert
trainable prompts with gates (for zero init).

Notes on the multi-adapter pattern:
- We store the states of different adapters by keeping a dictionary of AdaptedAttention modules indexed by adapter
  name.
- Every time we switch adapters, we remove the modules of the currently active adapter from the model, store them
  in the dictionary, and replace them with the modules of the new adapter.
- To avoid duplicated and potentially inconsistent state, the currently active adapter is always removed from the
  dictionary.
- Disabling the adapter would also result in the modules being removed from the model.
ÚconfigsÚadapter_namec                 ó  >• [         TU ]  5         Xl        0 U l        0 U l        0 U l        S U l        SU l        U R                  R                  U l        U R                  X2U   5        U R                  U R                  5        g )NT)ÚsuperÚ__init__ÚmodelÚpeft_configÚ_parentsÚ_cached_adaptersÚ_active_adapterÚ_enabledÚforwardÚadd_adapterÚ(_mark_only_adaption_prompts_as_trainable)Úselfr   r   r   Ú	__class__s       €Ú^/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/adaption_prompt/model.pyr   ÚAdaptionPromptModel.__init__*   sq   ø€ Ü‰ÑÔØŒ
à<>ˆÔð 57ˆŒà13ˆÔà#ˆÔàˆŒØ—z‘z×)Ñ)ˆŒØ×Ñ˜¨|Ñ'<Ô=Ø×5Ñ5°d·j±jÕAó    ÚconfigÚreturnNc                 ón  • [        X R                  5      nXR                  ;   a  [        SU S35      e/ nU R                  R	                  5        HU  u  pEUR                  SUR                   35      (       d  M*  [        U R                  U5      u  n  nUR                  U5        MW     [        U5      UR                  :  a%  [        SUR                   S[        U5       S35      eX2R                  * S nX0R                  U'   U R                  b,  U R                  (       a  U R                  U R                  5        Xl        X R                  U'   U R                  X#5        U R                  (       d  U R                  U R                  5        UR                   (       a  [#        U R                  U5        gg)z.Add an adapter with the given name and config.úAdapter with name 'z' already exists.Ú.z&Config specifies more adapter layers 'z' than the model has 'z'.N)r   r   r   Ú
ValueErrorÚnamed_modulesÚendswithÚtarget_modulesr   ÚappendÚlenÚadapter_layersr   r   r   Ú_remove_adapted_attentionsÚ_create_adapted_attentionsÚinference_moder   )r   r   r!   ÚparentsÚnameÚ_Úpars          r   r   ÚAdaptionPromptModel.add_adapter<   sv  € ä ¯
©
Ó3ˆØ×+Ñ+Ó+ÜÐ2°<°.Ð@QÐRÓSÐSàˆØ—z‘z×/Ñ/Ö1‰GˆDØ�}‰}˜q ×!6Ñ!6Ð 7Ð8×9Ó9Ü+¨D¯J©J¸Ó=‘	��Q˜Ø—‘˜sÖ#ñ 2ô ˆw‹<˜&×/Ñ/Ó/ÜØ8¸×9NÑ9NÐ8OÐOeÔfiÐjqÓfrÐesÐsuÐvóð ð ×0Ñ0Ð0Ð2Ð3ˆØ&-�‰�lÑ#ð ×ÑÑ+°··Ø×+Ñ+¨D×,@Ñ,@ÔAØ+ÔØ)/×Ñ˜Ñ&Ø×'Ñ'¨Ô8Ø�}�}Ø×+Ñ+¨D×,@Ñ,@ÔAà× × Ü˜DŸJ™J¨Õ5ð !r    r/   c                 ó  • U(       a  [        S5      eU R                  U:X  a  gXR                  ;  a  [        SU S35      eU R                  (       a,  U R	                  U R                  5        U R                  U5        Xl        g)z5Set the model to use the adapter with the given name.z8inference_mode is not supported for AdaptionPromptModel.Nr$   z' does not exist.)r&   r   r   r   r-   Ú_set_adapted_attentions)r   r   r/   s      r   Úset_adapterÚAdaptionPromptModel.set_adapter_   ss   € æÜÐWÓXÐXØ×Ñ <Ó/ØØ×/Ñ/Ó/ÜÐ2°<°.Ð@QÐRÓSÐSà�=�=Ø×+Ñ+¨D×,@Ñ,@ÔAØ×(Ñ(¨Ô6à+Õr    c                 óH   • SU l         U R                  U R                  5        g)zEEnable adapter layers by swapping in cached AdaptedAttention modules.TN)r   r6   r   ©r   s    r   Úenable_adapter_layersÚ)AdaptionPromptModel.enable_adapter_layersn   s   € àˆŒØ×$Ñ$ T×%9Ñ%9Õ:r    c                 óH   • SU l         U R                  U R                  5        g)z@Disable adapter layers by swapping out AdaptedAttention modules.FN)r   r-   r   r:   s    r   Údisable_adapter_layersÚ*AdaptionPromptModel.disable_adapter_layerss   s   € àˆŒØ×'Ñ'¨×(<Ñ(<Õ=r    r0   c           	      ó¤  • U HÊ  nU R                   R                  R                  S:X  aG  [        U R                   R                  R                  UR                  [        X1R                  5      S9nOF[        U R                   R                  R                  UR                  [        X1R                  5      S9n[        X1R                  U5        MÌ     g)zHWrap LlamaAttention modules with newly created AdaptedAttention modules.Úgpt2)Ú
model_typeÚadapter_lenr   N)	r   r!   rB   r	   rC   Úgetattrr)   r   Úsetattr)r   r!   r0   r3   Úattns        r   r.   Ú.AdaptionPromptModel._create_adapted_attentionsx   s    € ãˆCØ�z‰z× Ñ ×+Ñ+¨vÓ5Ü*Ø#Ÿz™z×0Ñ0×;Ñ;Ø &× 2Ñ 2Ü! #×'<Ñ'<Ó=ñ‘ô (Ø#Ÿz™z×0Ñ0×;Ñ;Ø &× 2Ñ 2Ü! #×'<Ñ'<Ó=ñ�ô
 �C×.Ñ.°Ö5ò r    c                 óÌ   • U R                   U   nU R                   U	 U R                  U   n[        U R                  U   5       H  u  pE[	        XSR
                  X$   5        M     g)zDReplace LlamaAttention modules with cached AdaptedAttention modules.N)r   r   Ú	enumerater   rE   r)   )r   r   Úcachedr!   Úir3   s         r   r6   Ú+AdaptionPromptModel._set_adapted_attentionsŠ   s\   € à×&Ñ& |Ñ4ˆØ×!Ñ! ,Ð/Ø×!Ñ! ,Ñ/ˆÜ §¡¨lÑ ;Ö<‰FˆAÜ�C×.Ñ.°±	Ö:ò =r    c                 óú   • U R                   U   n/ nU R                  U    HI  n[        XBR                  5      nUR	                  U5        [        XBR                  UR                  5        MK     X0R                  U'   g)zKRemove AdaptedAttention modules from the model and store them in the cache.N)r   r   rD   r)   r*   rE   r   r   )r   r   r!   Úadapted_attentionsr3   rF   s         r   r-   Ú.AdaptionPromptModel._remove_adapted_attentions’   sn   € à×!Ñ! ,Ñ/ˆØÐØ—=‘= Ô.ˆCÜ˜3× 5Ñ 5Ó6ˆDØ×%Ñ% dÔ+Ü�C×.Ñ.°·
±
Ö;ñ /ð /A×Ñ˜lÒ+r    r   c                 óh   • UR                  5        H  u  p#[        U5      (       a  M  SUl        M      g)z?Freeze all parameters of the model except the adaption prompts.FN)Únamed_parametersr
   Úrequires_grad)r   r   ÚnÚps       r   r   Ú<AdaptionPromptModel._mark_only_adaption_prompts_as_trainableœ   s)   € à×*Ñ*Ö,‰DˆAÜ/°×2Ó2Ø"'�–ò -r    r1   c                 ó~   >•  [         TU ]  U5      $ ! [         a     US:X  a  e [        U R                  U5      s $ f = f)z1Forward missing attributes to the wrapped module.r   )r   Ú__getattr__ÚAttributeErrorrD   r   )r   r1   r   s     €r   rW   ÚAdaptionPromptModel.__getattr__¢   sD   ø€ ð	-Ü‘7Ñ& tÓ,Ð,øÜó 	-ð �w‹ØÜ˜4Ÿ:™: tÓ,Ò,ð	-ús   ƒ ’'<»<)r   r   r   r   r   r   r   )F)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚdictÚstrr   r   r   Úboolr7   r;   r>   ÚlistÚnnÚModuler.   r6   r-   r   rW   Ú__static_attributes__Ú__classcell__)r   s   @r   r   r      sØ   ø† ñð B tð B¸3÷ Bð$!6¨ð !6Ð5Ið !6Èdô !6ñF,¨ð ,¸Tð ,Èdõ ,ò;ò
>ð
6Ð1Eð 6ÐPTÐUW×U^ÑU^ÑP_ð 6Ðdhô 6ð$;°Cð ;¸Dô ;ðA°sð A¸tô Að(¸b¿i¹ið (ÈDô (ð	- ÷ 	-õ 	-r    r   )Útorch.nnrc   Ú
peft.utilsr   r   r!   r   r   Úlayerr   r	   Úutilsr
   rd   r   © r    r   Ú<module>rl      s(   ðõ  ç 7ç 8ß 8Ý /ôR-˜"Ÿ)™)õ R-r    