ó
    >:jŸ'  ã                  óê   • S SK Jr  S SKrS SKrS SKJrJr  S SKrS SKJ	r	  S SK
JrJr  S SKJr  SSKJr  SS	KJrJr  \	" 5       (       a&  S S
KJr   " S S\R,                  R.                  \5      rSS jrg)é    )ÚannotationsN)ÚAnyÚOptional)Úis_hqq_available)ÚBaseTunerLayerÚcheck_adapters_to_merge)Ú	transposeé   )Ú
LoraConfig)Ú	LoraLayerÚLoraVariant)Ú	HQQLinearc                  óª   ^ • \ rS rSr  S           SU 4S jjjrSS jrSSS jjrSS jrS r          SS jr	SS jr
SU 4S	 jjrS
rU =r$ )ÚHqqLoraLinearé!   c                óò   >• UR                   (       a"  [        U R                  R                   S35      e[        TU ]  5         [        R
                  " X5        SU l        X l        U R                  UUUUS9  g )Nz0 does not support lora_bias yet, set it to FalseF)Ú
lora_alphaÚconfig)
Ú	lora_biasÚ
ValueErrorÚ	__class__Ú__name__ÚsuperÚ__init__r   Úfan_in_fan_outÚ_active_adapterÚupdate_layer)ÚselfÚ
base_layerÚadapter_namer   Úrr   Úkwargsr   s          €ÚQ/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/lora/hqq.pyr   ÚHqqLoraLinear.__init__#   sr   ø€ ð ××Ü  D§N¡N×$;Ñ$;Ð#<Ð<lÐ!mÓnÐnä‰GÑÔÜ×Ò˜tÔ0Ø"'ˆDÔà#/Ô Ø×ÑØØØ%Øð	 ò ó    c               ó@   • UR                   (       d  g SSKJn  U" 5       $ )Nr
   )ÚDoraLinearVariant)Úuse_doraÚvariantsr'   )r   r   r"   r'   s       r#   Úresolve_lora_variantÚ"HqqLoraLinear.resolve_lora_variant;   s   € Ø—?—?Øå3á$Ó&Ð&r%   c                óî  • [        X5      nU(       d  gU GH[  nX0R                  R                  5       ;  a  M#  U R                  5       n0 [        R
                  " UR                  5      ESUR                  0EnUR                  5       nX0R                  ;  a  U R                  U5      nXg-   nOU R                  U   R                  XU5      nU(       a8  [        R                  " U5      R                  5       (       d  [        SU S35      e[!        SXTR"                  UR$                  S9n	UR'                  SS5        U	R(                  " U40 UD6  X�l        U R,                  R/                  U5        GM^     g)a  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`list[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged.
        Defaults to `None`.
NÚoffload_metaz1NaNs detected in the merged weights. The adapter z seems to be broken©Úcompute_dtypeÚdevice)r   Úlora_AÚkeysÚget_base_layerÚcopyÚdeepcopyÚquant_configr-   Ú
dequantizeÚlora_variantÚget_delta_weightÚ
merge_safeÚtorchÚisfiniteÚallr   r   r/   r0   ÚpopÚquantizer   Úmerged_adaptersÚappend)
r   Ú
safe_mergeÚadapter_namesÚactive_adapterÚlayerr6   ÚoutputÚ	lora_dataÚw_dataÚnew_hqq_layers
             r#   ÚmergeÚHqqLoraLinear.mergeC   sG  € ô 4°DÓHˆMÞ àä"/�Ø!¯©×)9Ñ)9Ó);Ó;Ùà×+Ñ+Ó-�Øh¤$§-¢-°×0BÑ0BÓ"CÐhÀ^ÐUZ×UgÑUgÑh�à×)Ñ)Ó+�Ø!×):Ñ):Ó:Ø $× 5Ñ 5°nÓ E�IØ#Ñ/‘Fà!×.Ñ.¨~Ñ>×IÑIÈ$Ð`fÓg�Fæ¤e§n¢n°VÓ&<×&@Ñ&@×&BÑ&BÜ$ØKÈNÐK[Ð[nÐoóð ô !*¨$°×L_ÑL_Ðhm×htÑhtÑ u�Ø× Ñ  °Ô6Ø×&Ò& vÑ>°Ò>Ø"/”Ø×$Ñ$×+Ñ+¨N×;ò/ #0r%   c                óV  • U R                   (       d  [        R                  " S5        g[        U R                  5      S:”  Gag  U R                  R                  5       nXR                  R                  5       ;  a  MS  U R                  5       n0 [        R                  " UR                  5      ESUR                  0EnUR                  5       nXR                  ;  aI  U R                  U5      nUR!                  UR"                  5      R!                  UR$                  5      U-
  nOU R                  U   R'                  XU5      n[)        SX2R*                  UR$                  S9nUR                  SS5        UR,                  " U40 UD6  Xpl        [        U R                  5      S:”  a  GMf  gg)zG
This method unmerges all merged adapter layers from the base weights.
z Already unmerged. Nothing to do.Nr   r-   r.   )ÚmergedÚwarningsÚwarnÚlenr@   r>   r1   r2   r3   r4   r5   r6   r-   r7   r8   r9   ÚtoÚdtyper0   Úunmerger   r/   r?   r   )r   rD   rE   r6   rF   rG   rH   rI   s           r#   rS   ÚHqqLoraLinear.unmergen   sX  € ð —;—;Ü—’Ð@ÔAØä�d×*Ñ*Ó+¨aÔ/Ø!%×!5Ñ!5×!9Ñ!9Ó!;�Ø!¯©×)9Ñ)9Ó);Ó;Ùà×+Ñ+Ó-�Øh¤$§-¢-°×0BÑ0BÓ"CÐhÀ^ÐUZ×UgÑUgÑh�Ø×)Ñ)Ó+�à!×):Ñ):Ó:Ø $× 5Ñ 5°nÓ E�IØ#ŸY™Y y§¡Ó7×:Ñ:¸9×;KÑ;KÓLÈyÑX‘Fà!×.Ñ.¨~Ñ>×FÑFÀtÐ]cÓd�Fä )¨$°×L_ÑL_Ðhm×htÑhtÑ u�Ø× Ñ  °Ô6Ø×&Ò& vÑ>°Ò>Ø"/”ô% �d×*Ñ*Ó+¨a×/Ð/r%   c                óœ   • [        U R                  U   R                  U R                  U   R                  -  S5      U R                  U   -  $ )NF)r	   Úlora_BÚweightr1   Úscaling)r   Úadapters     r#   r9   ÚHqqLoraLinear.get_delta_weightŠ   sK   € äØ—K‘K Ñ(×/Ñ/°$·+±+¸gÑ2F×2MÑ2MÑMØóð —,‘,˜wÑ'ñ	(ðr%   c          	     óö  • U R                   " U/UQ70 UD6n[        U5      n/ nU H9  nUR                  [        U5       V	V
s/ s H  u  pšX¨:X  d  M  U	PM     sn
n	5        M;     [        U5       GH   u  p¼US:X  a  M  XÀR                  R                  5       ;  a  M-  U R                  U   nU R                  U   nU R                  U   nU R                  U   n[        R                  " 5       (       + nU(       a1  UR                  nU R                  XR                  R                  5      nXU      nU" U" U" U5      5      5      U-  nU(       a  UR                  W5      nXWU   ==   U-  ss'   GM     U$ s  sn
n	f )NÚ__base__)r   ÚsetrA   Ú	enumerater1   r2   rV   Úlora_dropoutrX   r;   Úis_autocast_enabledrR   Ú_cast_input_dtyperW   rQ   )r   ÚxrC   Úargsr"   ÚresultÚunique_adaptersÚsub_batch_indices_listrY   ÚindexÚitemÚirD   r1   rV   ÚdropoutrX   Úrequires_conversionÚexpected_dtypeÚ	sub_batchrF   s                        r#   Ú_mixed_batch_forwardÚ"HqqLoraLinear._mixed_batch_forward“   sd  € ð
 —_’_ QÐ8¨Ò8°Ñ8ˆFä! -Ó0ˆOØ%'Ð"Û*�Ø&×-Ñ-Ì	ÐR_ÔH`Ô.tÒH`¹¸ÐdhÑds¯uÑH`Ò.tÖuñ +ô &/¨×%?Ñ!�Ø! ZÓ/ÙØ!¯©×)9Ñ)9Ó);Ó;ÙàŸ™ ^Ñ4�ØŸ™ ^Ñ4�Ø×+Ñ+¨NÑ;�ØŸ,™, ~Ñ6�ä*/×*CÒ*CÓ*EÔ&EÐ#Þ&Ø%+§\¡\�NØ×.Ñ.¨q·-±-×2EÑ2EÓF�Að °QÑ7Ñ8�	Ù¡¡w¨yÓ'9Ó :Ó;¸gÑE�Þ&Ø#ŸY™Y ~Ó6�FØ¨aÑ0Ó1°VÑ;Ö1ñ- &@ð0 ˆMùó5 /us   ÁE5ÁE5c           	     óì  • U R                   " U/UQ70 UD6  UR                  SS 5      nU R                  (       a9  U R                  (       a  U R	                  5         U R
                  " U/UQ70 UD6nU$ Ub  U R                  " U/UQ7SU0UD6nU$ U R                  (       a  U R
                  " U/UQ70 UD6nU$ U R
                  " U/UQ70 UD6nU R                   GH  nX`R                  R                  5       ;  a  M#  U R                  U   nU R                  U   nU R                  U   n	U R                  U   n
[        R                  " 5       (       + nU(       a1  UR                  nU R!                  XR"                  R                  5      nX`R$                  ;  a  XX" U" U	" U5      5      5      U
-  -   nOU R$                  U   R'                  U UUUS9nU(       d  GM  UR)                  W5      nGM     U$ )NrC   )rD   rb   rd   )Ú_check_forward_argsr>   Údisable_adaptersrM   rS   r   rn   Úactive_adaptersr1   r2   rV   r_   rX   r;   r`   rR   ra   rW   r8   ÚforwardrQ   )r   rb   rc   r"   rC   rd   rD   r1   rV   rj   rX   rk   rl   s                r#   rt   ÚHqqLoraLinear.forward¹   sá  € Ø×$Ò$ QÐ8¨Ò8°Ò8Ø"ŸJ™J ¸Ó=ˆMà×$×$Ø—;—;Ø—L‘L”NØŸš¨Ð<¨TÒ<°VÑ<�ðD ˆMðC Ñ*Ø×2Ò2°1Ðc°tÒcÈ=ÐcÐ\bÑc�ð@ ˆMð? ——ØŸš¨Ð<¨TÒ<°VÑ<�ð< ˆMð9 Ÿš¨Ð<¨TÒ<°VÑ<�à&*×&:Õ&:�NØ%¯[©[×-=Ñ-=Ó-?Ó?Ù Ø!Ÿ[™[¨Ñ8�FØ!Ÿ[™[¨Ñ8�FØ"×/Ñ/°Ñ?�GØ"Ÿl™l¨>Ñ:�Gä.3×.GÒ.GÓ.IÔ*IÐ'Þ*Ø)/¯©˜Ø ×2Ñ2°1·m±m×6IÑ6IÓJ˜à%×->Ñ->Ó>Ø!'¨&±¹À»
Ó1CÓ*DÀwÑ*NÑ!N™à!%×!2Ñ!2°>Ñ!B×!JÑ!JØ Ø+9ØØ#)ð	 "Kð "˜÷ +Ñ*Ø!'§¡¨>Ó!:›ñ1 ';ð4 ˆMr%   c                ó*   >• [         TU ]  5       nSU-   $ )Nzlora.)r   Ú__repr__)r   Úrepr   s     €r#   rw   ÚHqqLoraLinear.__repr__ä   s   ø€ Ü‘'Ñ"Ó$ˆCØ˜S‘=Ð r%   )r   r   r   )r   r
   )r   útorch.nn.Moduler    Ústrr   r   r!   Úintr   r|   ÚreturnÚNone)r   r   r}   zOptional[LoraVariant])FN)rB   ÚboolrC   zOptional[list[str]]r}   r~   )r}   r~   )
rb   útorch.Tensorrc   r   rC   z	list[str]r"   r   r}   r€   )rb   r€   r}   r€   )r}   r{   )r   Ú
__module__Ú__qualname__Ú__firstlineno__r   r*   rJ   rS   r9   rn   rt   rw   Ú__static_attributes__Ú__classcell__)r   s   @r#   r   r   !   s¢   ø† ð Øð	à'ð	ð ð	ð ð		ð
 ð	ð ð	ð ÷	ð 	ô0	'ö)	<ôV	0ò8	ð$	Ø!ð$	Ø*-ð$	Ø>Gð$	ØSVð$	àô$	ôL)	÷V	!õ 	!r%   r   c                ó¾   • S n[        U [        5      (       a  U R                  5       nOU n[        5       (       a#  [        U[        5      (       a  [        XQ4SU0UD6nU$ )Nr   )Ú
isinstancer   r3   r   r   r   )Útargetr    r   r"   Ú
new_moduleÚtarget_base_layers         r#   Údispatch_hqqr‹   é   s[   € Ø€Jä�&œ.×)Ñ)Ø"×1Ñ1Ó3Ñà"Ðä×ÑœjÐ):¼I×FÑFÜ"Ð#4Ñ\È6Ð\ÐU[Ñ\ˆ
àÐr%   )rˆ   rz   r    r{   r   r   )Ú
__future__r   r4   rN   Útypingr   r   r;   Úpeft.import_utilsr   Úpeft.tuners.tuners_utilsr   r   Úpeft.utils.otherr	   r   r   rE   r   r   Úhqq.core.quantizer   ÚnnÚModuler   r‹   © r%   r#   Ú<module>r•      sQ   ðõ #ã Û ß  ã å .ß LÝ &å ß )ñ ×ÑÝ+ôE!˜Ÿ™Ÿ™¨ô E!õPr%   