>:j)SSKJr SSKrSSKrSSKJrJr SSKrSSKJ r SSK J r SSK J r Jr "SS\ 5r"SS \ R \5rg) ) annotationsN)AnyOptional) BufferDict)BaseTunerLayercheck_adapters_to_mergec\rSrSrSrSrS Sjr\S Sj5rS Sjr SSSjjr SSSjjr S r g ) DeloraLayer)delora_Adelora_B delora_lambda)rdelora_dropout delora_w_normc Xl0Ul[R"05Ul[R "05Ul[R "05Ul[R "05Ul[0SS9Ul SUl /Ul X l UR5n[U[R 5(a$UR"UR$sUlUlg['S[)U535e)NT) persistentFzUnsupported layer type ) base_layerrnn ModuleDictr ParameterDictr r rrr_disable_adaptersmerged_adapterskwargsget_base_layer isinstanceLinear in_features out_features ValueErrortype)selfrrbase_layer_mods U/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/delora/layer.py__init__DeloraLayer.__init__)s$ mmB/((, ((, --b1't<!&! ,,. nbii 0 02@2L2LnNiNi /D d/6tN7K6LMN Nc[R"URSS9SS9n[R"URSS9SS9n[R"X#- XV-- 5nX-U-nXR S5-nU$)zpCompute delta = B @ diag(delora_lambda/r / (||A_i||*||B^j||)) @ A, scaled by provided w_norm (per-input channel)dim-C6?minr)torchclampnorm diag_embed unsqueeze) ABrrw_normAnBndiagdeltas r$_compute_deltaDeloraLayer._compute_delta=su [[AD 1 [[AD 1 1RW =>1 ((++ r'c XR;dXR;a[SUS35eURURUURUURUUR UUR U5nU$)NzAdapter z not found.)r r r r;rrr)r"adapterr:s r$get_delta_weightDeloraLayer.get_delta_weightIs -- '7--+Gxy <= =## MM' " MM' "   w ' FF7O   w '   r'c US::a[SU35eX RU'[R"[R "X R 55URU'[R"[R "URU55URU'[R"[R "S55URU'US:a[R"US9nO[R"5nURR[R"X055 UR!XU5 UR#U5 UR%UR&US9 g)aiInternal function to create delora adapter Args: adapter_name (`str`): Name for the adapter to add. r (`int`): Rank for the added adapter. delora_lambda (`float`): Boundary for the adapter's norm. module_dropout (`float`): The dropout probability for disabling adapter during training. init_weights (`bool`): Whether to initialize weights. rz?`r` should be a positive integer value but the value passed is r)g)p)inference_modeN)r rr Parameterr/emptyrr rr rDropoutIdentityrupdaterreset_delora_parameters%_move_adapter_to_device_of_base_layer set_adapteractive_adapters) r" adapter_namerrmodule_dropout init_weightsrCrmodule_dropout_layers r$ update_layerDeloraLayer.update_layerVs& 6^_`^abc c |&(ll5;;qBRBR3S&T l#&(ll5;;t?P?PRS3T&U l#+-<< A+G<( C #%::#? #%;;=  ""2==,1U#VW $$\O 22<@ --nMr'cXRR5;agUSLal[RR URU[ R "S5S9 [RRURU5 O~[RR URU[ R "S5S9 [RR URU[ R "S5S9 URURR[U55 [R"5 UR5R nUR"R$S:wa.[R&"URSS9R)5nO,[R*"UR,SUR"S9nXPR.U'SSS5 g!,(df  g=f) NT)ametarr*r))device)r keysrinitkaiming_uniform_mathsqrtzeros_r rdatafill_floatr/no_gradrweightrWr!r1detachonesshaper)r"rMrOrwr6s r$rI#DeloraLayer.reset_delora_parameters}sY }}113 3  4  GG $ $T]]<%@DIIaL $ Q GGNN4==6 7 GG $ $T]]<%@DIIaL $ Q GG $ $T]]<%@DIIaL $ Q <(--33E-4HI]]_##%,,Axx}}&AFF299;AGGAJqxx@/5  | ,__s BG:: H) rrr r rrrrrrrrN)rz nn.ModulereturnNone) r4 torch.Tensorr5rjrrjrintr6rjrhrj)r>strrhrj)TF)rMrlrrkrr`rNr`rOboolrCrmrrrhri)Tg.@)rMrlrOrmrr`rhri) __name__ __module__ __qualname____firstlineno__adapter_layer_namesother_param_namesr% staticmethodr;r?rQrI__static_attributes__r'r$r r s  O(   ( 9E JM Wc     &"$%N%N %N %N  %N  %N%N%N %NT"# 666 6  66r'r c^\rSrSrS S U4SjjjrS S SjjrS SjrSSjrSSSjjrSU4Sjjr Sr U=r $) DeloraLinearc >[TU]5 [R"X40UD6 X lUR X#XEU5 g)N)superr%r _active_adapterrQ) r"rrMrrrNrOr __class__s r$r%DeloraLinear.__init__s; T88+ ,=,Wr'c[X5nU(dgUGH^nX0RR5;dM#UR5nUR U5R 5R URRURRS9n[R"5 U(aqURRR5nXe-n[R"U5R5(d[!SUS35eXdRl O%URRR#U5 SSS5 UR$R'U5 GMa g!,(df  N/=f)a Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`list[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. N)dtyperWz1NaNs detected in the merged weights. The adapter z seems to be broken)rr rXrr?rctorbrrWr/rar^cloneisfiniteallr add_rappend)r" safe_merge adapter_namesactive_adapterr delta_weight orig_weightss r$mergeDeloraLinear.merges.0D  +N!3!3!55!002 )).9VXRj//55j>O>O>V>VRW ]]_!'1'8'8'='='C'C'E '3'B $~~l;??AA","STbSccv w#2>))."))..33LA%$$++N;-,%_s 3BE:: F cUR(d[R"S5 g[UR5S:aURR 5nXR R5;a<UR5R=RURU5-sl [UR5S:aMgg)z: Unmerge all merged adapter layers from the base weights. z Already unmerged. Nothing to do.Nr) mergedwarningswarnlenrpopr rXrrbr^r?)r"rs r$unmergeDeloraLinear.unmerges{{ MM< = $&&'!+!11557N!3!3!55##%,,11T5J5J>5ZZ1$&&'!+r'c~URnUR(a9UR(aUR5 UR"U/UQ70UD6nGOUR(aUR"U/UQ70UD6nGOUR (d%UR"U/UQ70UD6R U5$UR"U/UQ70UD6n[R"U5nUR GHnXR;aMURU"U5n [RRXRU-URU5n [R"URUR!SS9SS9n [R"UR"UR!SS9SS9n UR$UUR&U- X-- n X-n [RRXR"U5n Xz- nGM XgR UR5-nUR U5nU$)Nr)r*r,r-r)rdisable_adaptersrrrrLrr/ zeros_liker rr functionallinearrr0r1r rr)r"xargsrprevious_dtyperesultbase_outadd_outr>x_dhr7r8scalings r$forwardDeloraLinear.forwards  {{ __Q888F [[__Q888F''q:4:6:==nMMq:4:6:H&&x0G//--/))'215MM((/A/A'/J)JDMMZaLbc[[w!7!> ::F>* r'cg)NTrv)r"rMs r$supports_lora_conversion%DeloraLinear.supports_lora_conversion sr'c*>[TU]5nSU-$)Nzdelora.)r{__repr__)r"repr}s r$rDeloraLinear.__repr__sg 3r')r|)T) rMrlrrkrr`rNr`rOrmrhri)FN)rrmrzOptional[list[str]]rhri)rhri)rrjrrrrrhrj)default)rMrlrhrm)rhrl) rnrorprqr%rrrrrru __classcell__)r}s@r$rxrxsw" X X  X  X  X X  X X'rsB#  /L|6.|6~v299kvr'