ó
    >:j‹B  ã                  ól   • S SK Jr  S SKJrJr  S SKJrJrJr  S SK	J
r
  S SKJr  \ " S S\
5      5       rg)	é    )Úannotations)Ú	dataclassÚfield)ÚLiteralÚOptionalÚUnion)Ú
PeftConfig)ÚPeftTypec                  ó&  ^ • \ rS rSr% Sr\" SSS0S9rS\S'   \" S	SS
0S9rS\S'   \" SSS0S9r	S\S'   \" SSS0S9r
S\S'   \" SSS0S9rS\S'   \" SSS0S9rS\S'   \" SSS0S9rS\S'   \" SSS0S9rS\S'   \" SSS 0S9rS!\S"'   \" S#SS$0S9rS\S%'   \" S#SS&0S9rS\S''   \" S(SS)0S9rS\S*'   \" SSS+0S9rS\S,'   \" S-SS.0S9rS/\S0'   \" S-SS10S9rS/\S2'   \" S3SS40S9rS/\S5'   \" SSS60S9rS!\S7'   \" S8SS90S9rS/\S:'   U 4S; jrS<rU =r$ )=ÚAdamssConfigé   aK  
Configuration class for Adamss (Adaptive Multi-Subspaces) method.

AdaMSS is a parameter-efficient fine-tuning method that decomposes weight matrices using SVD and clusters the
decomposed space into multiple trainable subspaces. It learns low-rank updates within these subspaces while keeping
the original weights frozen.

Args:
    r (`int`):
        Total rank for SVD decomposition (denoted as R in the paper). This determines how many singular vectors are
        used to represent the weight matrix before clustering. Higher values capture more information from the
        original weights but require more computation and memory. Lower values provide stronger regularization.
        Typical values range from 50 to 500. Default is 100.

    num_subspaces (`int`):
        Number of subspaces (K) to cluster the SVD-decomposed space into. Each subspace learns independent low-rank
        updates. Increasing this value allows finer-grained adaptation but increases the number of trainable
        parameters proportionally. When using ASA (Adaptive Subspace Allocation), this determines the initial
        number of subspaces before pruning. Typical values range from 3 to 10. Default is 5.

    subspace_rank (`int`):
        The rank (r_i) for each trainable subspace. This controls the capacity of each subspace to learn
        adaptations. Higher values increase expressiveness but also increase trainable parameters. Total trainable
        parameters scale as O(num_subspaces * subspace_rank * (in_dim + out_dim) / num_subspaces). For most tasks,
        values of 1-4 work well. Default is 1.

    target_modules (`Optional[Union[list[str], str]]`):
        The names of the modules to apply AdaMSS to. If specified, only these modules will be adapted. Can be a
        list of exact module names or a regex expression. For example, `['q_proj', 'v_proj']` for attention layers,
        or `'.*decoder.*(SelfAttention|EncDecAttention).*(q|v)$'` for regex matching.

    modules_to_save (`Optional[list[str]]`):
        List of modules apart from AdaMSS layers to be set as trainable and saved in the final checkpoint. These
        modules will be fully fine-tuned (not just low-rank). Required for randomly initialized heads like
        `classifier` or `score` in classification tasks.

    init_weights (`Literal["orthogonal"]`):
        Initialization method for AdaMSS trainable weights. Currently only "orthogonal" is supported, which uses
        orthogonal initialization for the B matrices (output projection). The A matrices are initialized to zero to
        ensure the model starts from the pretrained weights. Set to None to skip initialization when loading from a
        checkpoint. Default is "orthogonal".

    layers_to_transform (`Optional[Union[list[int], int]]`):
        Specific layer indices to apply AdaMSS to. If specified, only these layers will be adapted, useful for
        experimenting with which layers benefit most from adaptation. Can be a single integer or a list of
        integers.

    layers_pattern (`Optional[Union[list[str], str]]`):
        Pattern to match layer names when `layers_to_transform` is specified. Used to extract layer indices from
        module names that don't follow the common pattern.

    use_asa (`bool`):
        Whether to enable Adaptive Subspace Allocation (ASA). When enabled, ASA dynamically prunes less important
        subspaces during training based on gradient information, reducing the effective number of parameters while
        maintaining performance. Requires integration with a training callback. Default is False.

    asa_target_subspaces (`int`):
        Target total number of active subspaces across all layers when ASA is enabled. ASA will progressively prune
        subspaces until this target is reached. Lower values result in more aggressive pruning and fewer trainable
        parameters. Should be less than `num_subspaces * num_target_modules`. Typical values range from 20 to 100
        depending on model size. Default is 50.

    init_warmup (`int`):
        Number of training steps to wait before starting ASA pruning. During warmup, all subspaces remain active to
        allow importance scores to stabilize. Higher values give more time for accurate importance estimation but
        delay pruning. Typical values range from 50 to 200. Default is 50.

    final_warmup (`int`):
        Training step at which ASA completes pruning and reaches `asa_target_subspaces` active subspaces. The
        pruning is distributed between `init_warmup` and `final_warmup`. Should be set based on total training
        steps; typically 1/3 to 1/2 of total training steps. Default is 1000.

    mask_interval (`int`):
        Number of training steps between ASA mask updates. Lower values allow more frequent adaptation but increase
        overhead. Higher values provide more stable importance estimates between updates. Typical values range from
        50 to 200. Default is 100.

    asa_importance_beta (`float`):
        Exponential moving average (EMA) coefficient for smoothing subspace importance scores. Higher values
        (closer to 1.0) give more weight to historical importance, providing stability. Lower values make
        importance more responsive to recent gradients. Typical values range from 0.8 to 0.95. Default is 0.85.

    asa_uncertainty_beta (`float`):
        EMA coefficient for smoothing importance uncertainty estimates. Controls how quickly uncertainty adapts to
        gradient variance. Similar to asa_importance_beta, higher values provide more stable estimates. Typical
        values range from 0.8 to 0.95. Default is 0.85.

    asa_schedule_exponent (`float`):
        Schedule exponent controlling the decay rate from total subspaces to `asa_target_subspaces` during ASA
        warmup. Higher values result in faster initial pruning (more aggressive early reduction), while lower
        values provide a more gradual, linear-like decay. The formula is: current_active_subspaces =
        asa_target_subspaces + (asa_total_subspaces - asa_target_subspaces) * (progress ** exponent). Typical
        values range from 1.0 (linear) to 5.0 (aggressive). Default is 3.0.

    use_dynamic_rank (`bool`):
        Whether to dynamically determine subspace ranks based on singular value magnitudes. When True, each
        subspace's rank is determined by counting singular values above a threshold, allowing different subspaces
        to have different effective ranks. When False, all subspaces use the fixed `subspace_rank`. Default is
        False.

    svd_threshold (`float`):
        Threshold ratio for dynamic rank selection, only used when `use_dynamic_rank=True`. A singular value is
        considered significant if it exceeds `threshold * max_singular_value`. Higher values result in lower
        effective ranks (more aggressive truncation). Typical values range from 0.05 to 0.2. Default is 0.1 (10% of
        max).
éd   ÚhelpzÞTotal rank for SVD decomposition (R in the paper). Higher values capture more information but require more computation. The actual rank is clamped to min(r, in_features, out_features). Typical values: 50-500. Default: 100.)ÚdefaultÚmetadataÚintÚré   zßNumber of subspaces (K) to cluster the SVD space into. Each subspace learns independent low-rank updates. Increasing this allows finer-grained adaptation but increases trainable parameters. Typical values: 3-10. Default: 5.Únum_subspacesé   z Rank (r_i) for each trainable subspace. Higher values increase expressiveness but also increase parameters. For most tasks, values of 1-4 work well. Default: 1.Úsubspace_rankNzæList of module names or regex expression of the module names to replace with AdaMSS. For example, ['q_proj', 'v_proj'] or '.*decoder.*(SelfAttention|EncDecAttention).*(q|v)$'. For Vision Transformers, typically ['query', 'value'].zOptional[Union[list[str], str]]Útarget_modulesÚ
orthogonala1  Initialization method for AdaMSS trainable weights. Currently only 'orthogonal' is supported, which uses orthogonal initialization for B matrices. A matrices are initialized to zero to start from pretrained weights. Set to None to skip initialization when loading from a checkpoint. Default: 'orthogonal'.zOptional[Literal['orthogonal']]Úinit_weightsa  List of modules apart from AdaMSS layers to be set as trainable and saved in the final checkpoint. For example, in Sequence Classification or Token Classification tasks, the final layer `classifier/score` are randomly initialized and as such need to be trainable and saved.zOptional[list[str]]Úmodules_to_savezîSpecific layer indices to apply AdaMSS to. If specified, only these layers will be adapted, useful for experimenting with which layers benefit most. Can be a single integer or a list of integers. Default: None (adapt all matching layers).zOptional[Union[list[int], int]]Úlayers_to_transformz˜Pattern to match layer names when `layers_to_transform` is specified. Used to extract layer indices from module names that don't follow common patterns.Úlayers_patternFa  Whether to enable Adaptive Subspace Allocation (ASA). When enabled, ASA dynamically prunes less important subspaces during training based on gradient information, reducing parameters while maintaining performance. Requires a training callback. Default: False.ÚboolÚuse_asaé2   zäTarget total number of active subspaces across all layers when ASA is enabled. ASA progressively prunes subspaces until this target is reached. Lower values result in more aggressive pruning. Typical values: 20-100. Default: 50.Úasa_target_subspaceszâTraining steps to wait before starting ASA pruning. During warmup, all subspaces remain active to allow importance scores to stabilize. Higher values give more time for accurate estimation. Typical values: 50-200. Default: 50.Úinit_warmupiè  z¯Training step at which ASA completes pruning and reaches asa_target_subspaces. Should be set based on total training steps; typically 1/3 to 1/2 of total steps. Default: 1000.Úfinal_warmupz¾Training steps between ASA mask updates. Lower values allow more frequent adaptation but increase overhead. Higher values provide more stable estimates. Typical values: 50-200. Default: 100.Úmask_intervalg333333ë?a  EMA coefficient for smoothing subspace importance scores during ASA. Higher values (closer to 1.0) give more weight to historical importance, providing stability. Lower values make importance more responsive to recent gradients. Typical values: 0.8-0.95. Default: 0.85.ÚfloatÚasa_importance_betazÞEMA coefficient for smoothing importance uncertainty estimates during ASA. Controls how quickly uncertainty adapts to gradient variance. Higher values provide more stable estimates. Typical values: 0.8-0.95. Default: 0.85.Úasa_uncertainty_betag      @a}  Schedule exponent controlling the decay rate from total subspaces to asa_target_subspaces. Higher values result in faster initial pruning (aggressive early reduction), lower values provide gradual linear-like decay. Formula: current_active_subspaces = asa_target_subspaces + (total - target) * (progress ** exponent). Typical values: 1.0 (linear) to 5.0 (aggressive). Default: 3.0.Úasa_schedule_exponentaA  Whether to dynamically determine subspace ranks based on singular value magnitudes. When True, each subspace's rank is determined by counting singular values above a threshold, allowing different subspaces to have different effective ranks. When False (default), all subspaces use the fixed subspace_rank. Default: False.Úuse_dynamic_rankgš™™™™™¹?a!  Threshold ratio for dynamic rank selection (only used when use_dynamic_rank=True). A singular value is significant if it exceeds threshold * max_singular_value. Higher values result in lower effective ranks (more aggressive truncation). Typical values: 0.05-0.2. Default: 0.1 (10% of max).Úsvd_thresholdc                óŽ  >• [         R                  U l        [        U R                  [
        5      (       a  [        U R                  5      OU R                  U l        [        TU ]!  5         U R                  SL a  S U l	        U R                  S;  a  [        SU R                   35      e SS Kng ! [         a    [        S5      ef = f)NF)r   Nz/init_weights must be 'orthogonal' or None, got r   zUscikit-learn is required for AdaMSS. Please install it with: pip install scikit-learn)r
   ÚADAMSSÚ	peft_typeÚ
isinstancer   ÚlistÚsetÚsuperÚ__post_init__r   Ú
ValueErrorÚsklearnÚImportError)Úselfr4   Ú	__class__s     €ÚV/home/mande/repo/quber/.venv/lib/python3.13/site-packages/peft/tuners/adamss/config.pyr2   ÚAdamssConfig.__post_init__A  sµ   ø€ Ü!Ÿ™ˆŒä(2°4×3FÑ3FÌ×(MÑ(MŒC�×#Ñ#Ô$ÐSW×SfÑSfð 	Ôô 	‰ÑÔð ×Ñ Ò%Ø $ˆDÔð ×ÑÐ$8Ó8ÜÐNÈt×O`ÑO`ÐNaÐbÓcÐcð	wÜøÜó 	wÜÐuÓvÐvð	wús   Â)B. Â.C)r   r-   r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r   Ú__annotations__r   r   r   r   r   r   r   r   r!   r"   r#   r$   r&   r'   r(   r)   r*   r2   Ú__static_attributes__Ú__classcell__)r7   s   @r8   r   r      s¦  ø‡ ñiñV Øàð[ð
ñ	€A€só 	ñ ØàðXð
ñ	€M�3ó 	ñ Øàðeð
ñ€M�3ó ñ 7<ØàðIð
ñ	7€NÐ3ó 	ñ 5:ØàðXð
ñ
5€LÐ1ó 
ñ ,1Øàðzð
ñ	,€OÐ(ó 	ñ <AØàðcð
ñ	<ÐÐ8ó 	ñ 7<Øàð]ð
ñ7€NÐ3ó ñ Øàð"ð
ñ
€GˆTó 
ñ !&ØàðZð
ñ	!Ð˜#ó 	ñ ØàðUð
ñ	€K�ó 	ñ Øàð!ð
ñ	€L�#ó 	ñ Øàð8ð
ñ	€M�3ó 	ñ "'Øàð;ð
ñ
"Ð˜ó 
ñ #(ØàðZð
ñ	#Ð˜%ó 	ñ $)ØàðRð
ñ$Ð˜5ó ñ #Øàðcð
ñ
Ð�dó 
ñ !ØàðGð
ñ
€M�5ó 
÷wó wó    r   N)Ú
__future__r   Údataclassesr   r   Útypingr   r   r   Úpeft.configr	   Ú
peft.utilsr
   r   © rB   r8   Ú<module>rI      s:   ðõ #ç (ß +Ñ +å "Ý ð ô{w�:ó {wó ñ{wrB   