ó
    >:jR;  ã                  óä   • S SK Jr  S SKrS SKrS SKJr  S SKrS SKJr  S SK	J
r
  S SKJr  S SKJrJr  S SKJrJr  S SKJr  S	S
KJr  S	SKJr  SSKJr  SSKJrJr        SS jr " S S\5      rg)é    )ÚannotationsN)ÚUnion)Úis_bf16_available)ÚConv1D)Úis_bnb_4bit_availableÚis_bnb_available)Ú	BaseTunerÚBaseTunerLayer)Ú6TRANSFORMERS_MODELS_TO_RANDLORA_TARGET_MODULES_MAPPINGé   )Ú
BufferDict)Ú _maybe_include_all_linear_layersé   )ÚRandLoraConfig)ÚLinearÚRandLoraLayerc                ó”  • [        U [        5      (       aC  [        R                  " U [	        5       (       a  [        R
                  O[        R                  S9nOU n[        R                  " 5          [        R                  R                  R                  U[        R                  " S5      US9nUsSSS5        $ ! , (       d  f       g= f)a˜  
Kaiming Uniform Initialisation adapted to accept a `torch.Generator` object for PRNG.

Args:
    tensor_or_shape (`Union[torch.Tensor, tuple[int, ...]]`):
        Tensor to initialise, or shape of new tensor to create and then initialise.
    generator: (`torch.Generator`):
        Generator object that manages the state of the PRNG algorithm in use.

Returns:
    `torch.Tensor`: The initialised tensor.
)Údtypeé   )ÚaÚ	generatorN)Ú
isinstanceÚtupleÚtorchÚemptyr   Úbfloat16Úfloat16Úno_gradÚnnÚinitÚkaiming_uniform_ÚmathÚsqrt)Útensor_or_shaper   ÚtensorÚbasiss       ÚW/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/randlora/model.pyÚ_kaiming_initr(   &   s|   € ô  �/¤5×)Ñ)Ü—’ØÜ$5×$7Ñ$7”%—.’.¼U¿]¹]ñ
‰ð
 !ˆä	�Š�Ü—‘—‘×.Ñ.¨v¼¿ºÀ1»ÐQZÐ.Ð[ˆØ÷ 
��ús   Á0?B9Â9
Cc                  óŠ   ^ • \ rS rSr% SrSrS\S'   \r\	r
SS jrSSS jjrSS jrSS	 jrSU 4S
 jjrS r\S 5       rSrU =r$ )ÚRandLoraModeléC   aâ  
Creates a RandLoRA model from a pretrained transformers model.

Args:
    model ([`~transformers.PreTrainedModel`]): The model to be adapted.
    config ([`RandLoraConfig`]): The configuration of the RandLora model.
    adapter_name (`str`): The name of the adapter, defaults to `"default"`.
    low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
        Create empty adapter weights on meta device. Useful to speed up the loading process.

Returns:
    `torch.nn.Module`: The RandLora model.

Example:

    ```py
    >>> from transformers import AutoModelForCausalLM
    >>> from peft import RandLoraConfig, get_peft_model

    >>> base_model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m")
    >>> config = RandLoraConfig(r=32)
    >>> model = get_peft_model(base_model, config)
    ```

**Attributes**:
    - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
    - **peft_config** ([`RandLoraConfig`]): The configuration of the RandLora model.
Ú	randlora_ÚstrÚprefixc                ó²  • U R                  U R                  5      nU R                  X5      n[        X0R                  5      nSnU R                  R	                  5        Hå  u  pVU R                  X55      (       d  M  [        U[        R                  5      (       a  UR                  UR                  4nOg[        U[        5      (       aP  [        UR                  S5      (       a  UR                  R                  OUR                  R                  nUSSS2   nOM¼  Uc  UnMÃ  Xt:w  d  MÊ  [!        S [#        XG5       5       5      nMç     Uc  Sn[%        U5      eU$ )zÈ
Finds the largest input and output dimensions across linear layers that have been wrapped with RandLora.

This will be used for determining the size of the shared randlora_A and randlora_B matrices.
NÚds_shapeéÿÿÿÿc              3  ó<   #   • U  H  u  p[        X5      v •  M     g 7f)N)Úmax)Ú.0r   Úbs      r'   Ú	<genexpr>Ú*RandLoraModel._find_dim.<locals>.<genexpr>‚   s   é € Ð%]Ò<\±D°A¤c¨!§i iÒ<\ùs   ‚z_No layers types compatible with RandLora were found. Please check `peft_config.target_modules`.)Úget_model_configÚmodelÚ_prepare_adapter_configr   Únamed_modulesÚ_check_target_module_existsr   r   r   Úout_featuresÚin_featuresr   ÚhasattrÚweightr0   Úshaper   ÚzipÚ
ValueError)	ÚselfÚconfigÚmodel_configÚpeft_configÚlargest_shapeÚkeyÚmoduleÚmodule_shapeÚmsgs	            r'   Ú	_find_dimÚRandLoraModel._find_dime   s"  € ð ×,Ñ,¨T¯Z©ZÓ8ˆà×2Ñ2°6ÓHˆÜ6°{ÇJÁJÓOˆàˆØŸ:™:×3Ñ3Ö5‰KˆCØ×3Ñ3°K×EÑEÙä˜&¤"§)¡)×,Ñ,Ø%×2Ñ2°F×4FÑ4FÐF‘Ü˜F¤F×+Ñ+Ü9@ÀÇÁÐPZ×9[Ñ9[˜vŸ}™}×5Ò5Ðag×anÑan×atÑat�Ø+©D¨b¨DÑ1‘áàÑ$Ø ,�ÙàÕ,Ü %Ñ%]¼CÀÔ<\Ó%]Ó ]’ñ# 6ð& Ñ ØsˆCÜ˜S“/Ð!àÐó    c                ó‚  • U R                  U5      u  pE[        XE5      [        XE5      pv[        0 UR                  S9U l        [        0 UR                  S9U l        [        R                  " SS9R                  UR                  5      n[        R                  " UR                  SU4US9n	XqR                  -  n
U
R                  5       (       a  [        U
5      O[        U
5      S-   n
[        R                  " XjUR                  4US9n[        R                  " UR                   5      n[        R                  " U	R                   5      nSXËSSU-  -  :  '   SXËSSSU-  -  -
  :„  '   SXÙSSU-  -  :  '   SXÙSSSU-  -  -
  :„  '   XÝR#                  5       -  XÌR#                  5       -  p¹X�R
                  U'   X°R                  U'   g)	z^
Sparse random projections as described in https://cs-people.bu.edu/evimaria/cs565/kdd-rp.pdf
©Ú
persistentÚcpu©Údevicer   ©r   r1   r   N)rM   r3   Úminr   Úsave_projectionÚ
randlora_AÚ
randlora_Br   Ú	GeneratorÚmanual_seedÚprojection_prng_keyÚrandÚrÚ
is_integerÚintÚzerosrA   Ústd)rD   rE   Úadapter_nameÚsparsityÚlinear_out_dimÚlinear_in_dimÚmax_dimÚmin_dimr   rY   Ú	num_basesrZ   Úrandlora_B_sparseÚrandlora_A_sparses                 r'   Ú"_init_randlora_A_randlora_B_sparseÚ0RandLoraModel._init_randlora_A_randlora_B_sparseŠ   s©  € ð
 )-¯©°vÓ(>Ñ%ˆÜ˜~Ó=¼sÀ>Ó?a�ô % R°F×4JÑ4JÑKˆŒÜ$ R°F×4JÑ4JÑKˆŒô —O’O¨5Ñ1×=Ñ=¸f×>XÑ>XÓYˆ	ô —Z’Z §¡¨1¨gÐ 6À)ÑLˆ
ð Ÿh™hÑ&ˆ	Ø&/×&:Ñ&:×&<Ñ&<”C˜	”NÄ#ÀiÃ.ÐSTÑBTˆ	Ü—Z’Z °V·X±XÐ >È)ÑTˆ
ô "ŸKšK¨
×(8Ñ(8Ó9ÐÜ!ŸKšK¨
×(8Ñ(8Ó9ÐØ=?Ð q¨A°©LÑ'9Ñ9Ñ:ØABÐ q¨1°°H±Ñ+=Ñ'=Ñ=Ñ>Ø=?Ð q¨A°©LÑ'9Ñ9Ñ:ØABÐ q¨1°°H±Ñ+=Ñ'=Ñ=Ñ>ð × 5Ñ 5Ó 7Ñ7Ø× 5Ñ 5Ó 7Ñ7ð ð )3�‰˜Ñ%Ø(2�‰˜Ò%rO   c           
     óÜ  • U R                  U5      u  p4[        X45      [        X45      pe[        0 UR                  S9U l        [        0 UR                  S9U l        [        R                  " SS9R                  UR                  5      n[        UR                  SU4US9n[        X45      UR                  -  n	U	R                  5       (       a  [        U	5      O[        U	5      S-   n	[        R                  " [!        U	5       V
s/ s H  n
[        USUR                  4US9PM     sn
SS9nXˆR#                  5       -  X»R#                  5       -  p¸X€R
                  U'   X°R                  U'   g s  sn
f )NrQ   rS   rT   r   rV   )Údim)rM   r3   rW   r   rX   rY   rZ   r   r[   r\   r]   r(   r_   r`   ra   ÚcatÚrangerc   )rD   rE   rd   rf   rg   rh   ri   r   rY   rj   Ú_rZ   s               r'   Ú_init_randlora_A_randlora_BÚ)RandLoraModel._init_randlora_A_randlora_B³   s8  € Ø(,¯©°vÓ(>Ñ%ˆÜ˜~Ó=¼sÀ>Ó?a�ô % R°F×4JÑ4JÑKˆŒÜ$ R°F×4JÑ4JÑKˆŒô —O’O¨5Ñ1×=Ñ=¸f×>XÑ>XÓYˆ	ô # F§H¡H¨a°Ð#9ÀYÑOˆ
ô ˜Ó6¸¿¹ÑAˆ	Ø&/×&:Ñ&:×&<Ñ&<”C˜	”NÄ#ÀiÃ.ÐSTÑBTˆ	Ü—Y’YÜQVÐW`ÔQaÓbÒQaÈAŒ]˜G Q¨¯©Ð1¸YÔGÑQaÑbÐhiñ
ˆ
ð
 ",¯n©nÓ.>Ñ!>À
Ï^É^ÓM]Ñ@]�JØ(2�‰˜Ñ%Ø(2�‰˜Ò%ùò cs   Ä!E)c           
     ó  • UR                   (       aA  U R                  U5      u  pEU R                  X#[        R                  " [        XE5      5      S9  g UR                  (       a  U R                  X#SS9  g U R                  X#5        g )N)re   é   )Úvery_sparserM   rm   r"   r#   rW   Úsparsert   )rD   r9   rE   rd   rf   rg   s         r'   Ú_pre_injection_hookÚ!RandLoraModel._pre_injection_hookÎ   sm   € Ø××Ø,0¯N©N¸6Ó,BÑ)ˆNØ×3Ñ3Ø¬t¯yªy¼¸^Ó9[Ó/\ð 4ò ð �]�]Ø×3Ñ3°FÐSTÐ3ÒUà×,Ñ,¨VÕBrO   c                ó²  >• [         TU ]  U5        U R                  R                  5        HJ  nX!L a  M	  UR                  UR                  :w  d  M%  [        SUR                  < SUR                   S35      e   [        U R                  R                  5        Vs1 s H  oR                  iM     sn5      n[        U5      S:”  a  [        SU 35      egs  snf )z´
A helper method to check the config when a new adapter is being added.

Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

zcRandLora PRNG initialisation key must be the same for all adapters. Got config.projection_prng_key=z but previous config had Ú.r   zgRandLora projection weights must be saved for all adapters or none, but got multiple different values: N)	ÚsuperÚ_check_new_adapter_configrG   Úvaluesr]   rC   ÚsortedrX   Úlen)rD   rE   Úexisting_configÚsave_project_unique_valuesÚ	__class__s       €r'   r   Ú'RandLoraModel._check_new_adapter_configÙ   sä   ø€ ô 	‰Ñ)¨&Ô1à#×/Ñ/×6Ñ6Ö8ˆOØÒ(áà×2Ñ2°f×6PÑ6PÕPÜ ØzÐ_e×_yÑ_yÑ^{ð |+Ø+:×+NÑ+NÐ*OÈqðRóð ñ  9ô &,ÐRV×RbÑRb×RiÑRiÔRkÓ,lÒRkÈ×-CÔ-CÑRkÑ,lÓ%mÐ"ÜÐ)Ó*¨QÓ.ÜØyØ-Ð.ð0óð ð /ùò -ms   ÂCc                ó®  • Uc  [        S5      eUR                  n[        US5      =(       a    UR                  S Ln	UUR                  UR
                  UR                  UR                  [        U R                  SS5      [        U R                  SS5      S.n
XšS'   [        U[        5      (       aJ  UR                  UU R                  U R                  UUR                  UR
                  UR                  5        g U R                  " XR                  U R                  X#40 U
D6nX R                   ;  a  UR#                  S5        U R%                  XTX³5        g )NzCurrent Key shouldn't be `None`ÚbiasÚis_loaded_in_8bitFÚis_loaded_in_4bit)r_   Úrandlora_alphaÚrandlora_dropoutÚfan_in_fan_outÚinit_weightsÚloaded_in_8bitÚloaded_in_4bit)rC   r_   r?   rˆ   r‹   rŒ   r�   rŽ   Úgetattrr9   r   r   Úupdate_layerrY   rZ   Ú_create_new_moduleÚactive_adapterÚrequires_grad_Ú_replace_module)rD   Úrandlora_configrd   ÚtargetÚtarget_nameÚparentÚcurrent_keyÚoptional_kwargsr_   rˆ   ÚkwargsÚ
new_modules               r'   Ú_create_and_replaceÚ!RandLoraModel._create_and_replaceô   s3  € ð ÑÜÐ>Ó?Ð?à×ÑˆÜ�v˜vÓ&×B¨6¯;©;¸dÐ+BˆàØ-×<Ñ<Ø /× @Ñ @Ø-×<Ñ<Ø+×8Ñ8Ü% d§j¡jÐ2EÀuÓMÜ% d§j¡jÐ2EÀuÓMñ
ˆð ˆv‰Ü�fœf×%Ñ%Ø×ÑØØ—‘Ø—‘ØØ×.Ñ.Ø×0Ñ0Ø×,Ñ,õð ×0Ò0Ø§¡°$·/±/À<ñØ[añˆJð ×#6Ñ#6Ó6à×)Ñ)¨%Ô0Ø× Ñ  °jÕIrO   c                óÌ  • [        5       (       a
  SS KnSSKJn  [	        5       (       a  SSKJn  UR                  SS5      n	UR                  SS5      n
UR                  SS5      n[        U[        5      (       a  UR                  5       nOUnU
(       a†  [        UWR                  R                  5      (       aa  UR                  5       nUR                  UR                  R                  UR                  R                   UR"                  S	.5        W" XCX40 UD6$ U(       a†  [        UWR                  R
                  5      (       aa  UR                  5       nUR                  UR$                  UR&                  R(                  UR&                  R*                  S
.5        W" XCX40 UD6$ [        U[,        R                  R.                  5      (       a-  US   (       a"  [0        R2                  " S5        S=US'   U l        OV[        U[6        5      (       a2  SUS'   US   (       d"  [0        R2                  " S5        S=US'   U l        O[9        SU S35      e[/        UUUU4SU	0UD6nU$ )Nr   r   )ÚLinear8bitLt)Ú
Linear4bitrˆ   Fr�   r�   )Úhas_fp16_weightsÚ	thresholdÚindex)Úcompute_dtypeÚcompress_statisticsÚ
quant_typer�   zjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.TÚis_target_conv_1d_layerzafan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.zTarget module z is not supported. Currently, only the following modules are supported: `torch.nn.Linear`, `transformers.pytorch_utils.Conv1D`.)r   ÚbitsandbytesÚbnbr¢   r   r£   ÚpopÚgetr   r
   Úget_base_layerr   ÚcopyÚupdateÚstater¤   r¥   r¦   r§   r@   r¨   r©   r   r   ÚwarningsÚwarnr�   r   rC   )r—   rY   rZ   rd   r˜   r�   r¬   r¢   r£   rˆ   r�   r�   Útarget_base_layerÚeightbit_kwargsÚfourbit_kwargsrž   s                   r'   r“   Ú RandLoraModel._create_new_module   s?  € ô ×ÑÛ&å)ä ×"Ñ"Ý'à�z‰z˜& %Ó(ˆØŸ™Ð$4°eÓ<ˆØŸ™Ð$4°eÓ<ˆä�fœn×-Ñ-Ø &× 5Ñ 5Ó 7Ñà &ÐæœjÐ):¸C¿F¹F×<OÑ<O×PÑPØ$Ÿk™k›mˆOØ×"Ñ"à(9×(?Ñ(?×(PÑ(PØ!2×!8Ñ!8×!BÑ!BØ.×4Ñ4ñôñ   °jÑ`ÐP_Ñ`Ð`Þ¤
Ð+<¸c¿f¹f×>OÑ>O× PÑ PØ#Ÿ[™[›]ˆNØ×!Ñ!à%6×%DÑ%DØ+<×+CÑ+C×+WÑ+WØ"3×":Ñ":×"EÑ"Eñôñ ˜f°JÑ]ÈnÑ]Ð]ÜÐ)¬5¯8©8¯?©?×;Ñ;ØÐ&×'Ü—’ð7ôð MRÐQ�Ð'Ñ(¨?Ô+IøÜÐ)¬6×2Ñ2Ø04ˆFÐ,Ñ-ØÐ*×+Ü—’Øwôð MQÐP�Ð'Ñ(¨?Ô+IøäØ   ð )Jð Jóð ô ØØØØñ	
ð
 ð
ð ñ
ˆ
ð ÐrO   )rY   rZ   )Úreturnztuple[int, int])rw   )rE   r   rd   r-   re   ra   r¹   ÚNone)rE   r   rd   r-   r¹   rº   )r9   z	nn.ModulerE   r   rd   r-   r¹   rº   )rE   r   r¹   rº   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r.   Ú__annotations__r   Útuner_layer_clsr   Útarget_module_mappingrM   rm   rt   rz   r   rŸ   Ústaticmethodr“   Ú__static_attributes__Ú__classcell__)r…   s   @r'   r*   r*   C   sX   ø‡ ñð: €FˆCÓØ#€OØRÐô#öJ'3ôR3ô6	C÷ò6*JðX ñCó öCrO   r*   )r$   z$Union[torch.Tensor, tuple[int, ...]]r   ztorch.Generatorr¹   ztorch.Tensor) Ú
__future__r   r"   r³   Útypingr   r   Útorch.nnr   Úaccelerate.utils.importsr   Útransformers.pytorch_utilsr   Úpeft.import_utilsr   r   Úpeft.tuners.tuners_utilsr	   r
   Ú
peft.utilsr   Ú_buffer_dictr   Útuners_utilsr   rE   r   Úlayerr   r   r(   r*   © rO   r'   Ú<module>rÒ      se   ðõ #ã Û Ý ã Ý Ý 6Ý -ç Eß >õõ &Ý ;Ý "ß (ðØ9ðàðð ôô:a�Iõ arO   