ó
    >:jµ  ã                  óŠ   • S SK Jr  S SKrS SKJr  S SKJrJr  S SKJ	r	  S SK
r
\R                  " \5      r\ " S S5      5       rg)é    )ÚannotationsN)ÚCallable)Ú	dataclassÚfield)ÚAnyc                  ó¸   • \ rS rSr% SrS\S'   \" S S9rS\S'   \" \S	S
9r	S\S'   \" \S	S
9r
S\S'   \" \S	S	S9rS\S'   SS jrSS jrSS jrSS jrSS jrSrg)ÚBaseDataCollatoré   ad  Base data collator for Sentence Transformers models.

Preprocesses text columns via ``preprocess_fn`` (typically ``model.preprocess``),
producing ``{column}_input_ids``, ``{column}_attention_mask``, etc.  Handles prompt
resolution (per-column or per-dataset) and Router task mapping.

It is important that the columns are in the expected order. For example, if your dataset has columns
"answer", "question" in that order, then the MultipleNegativesRankingLoss will consider
"answer" as the anchor and "question" as the positive, and it will (unexpectedly) optimize for
"given the answer, what is the question?".
r   Úpreprocess_fnc                 ó
   • / SQ$ )N)ÚlabelÚlabelsÚscoreÚscores© r   ó    Úe/home/mande/repo/quber/.venv/lib/python3.13/site-packages/sentence_transformers/base/data_collator.pyÚ<lambda>ÚBaseDataCollator.<lambda>   s   € ÓCir   )Údefault_factoryú	list[str]Úvalid_label_columnsF)r   Úreprz1dict[str, str] | dict[str, dict[str, str]] | NoneÚrouter_mappingz7str | dict[str, str] | dict[str, dict[str, str]] | NoneÚprompts)r   Úinitr   zset[tuple[str, ...]]Ú_warned_columnsc                ó  • U R                   nU(       ab  [        U[        5      (       aM  [        [        [	        UR                  5       5      5      [        5      (       a  SU;   a  US   U;   a  X!S      $ 0 $ U=(       d    0 $ )zRResolve the router mapping for this batch, handling nested (per-dataset) mappings.Údataset_name)r   Ú
isinstanceÚdictÚnextÚiterÚvalues)ÚselfÚbatchr   s      r   Ú_resolve_router_mappingÚ(BaseDataCollator._resolve_router_mapping"   so   € à×,Ñ,ˆæÜ˜>¬4×0Ñ0Üœ4¤ ^×%:Ñ%:Ó%<Ó =Ó>Ä×EÑEà Ó&¨5°Ñ+@ÀNÓ+RØ%¨NÑ&;Ñ<Ð<ØˆIØ×# Ð#r   c                óZ  • U R                   n[        U[        5      (       a  U(       d  U$ SU;   nU(       a  US   U;   a  X!S      $ [        [        [	        UR                  5       5      5      [        5      (       a/  U(       d&  [        S[        UR                  5       5      < 35      e0 $ U$ )zKResolve the prompts for this batch, handling nested (per-dataset) mappings.r   a=  The prompts provided to the trainer are a nested dictionary. In this setting, the first level of the dictionary should map to dataset names and the second level to column names. However, as the provided dataset is a not a DatasetDict, no dataset names can be inferred. The keys to the provided prompts dictionary are )	r   r    r!   r"   r#   r$   Ú
ValueErrorÚlistÚkeys)r%   r&   r   Úis_multi_datasets       r   Ú_resolve_promptsÚ!BaseDataCollator._resolve_prompts/   s¢   € à—,‘,ˆÜ˜'¤4×(Ñ(¶ØˆNà)¨UÑ2ÐÞ  nÑ 5¸Ó @Ø Ñ0Ñ1Ð1ä”dœ4 §¡Ó 0Ó1Ó2´D×9Ñ9Þ#Ü ðGô HLÈGÏLÉLËNÓG[ÑF^ð`óð ð ˆIàˆr   c                ón   • [        U[        5      (       a  U$ [        U[        5      (       a	  X!;   a  X   $ g)z,Get the prompt string for a specific column.N)r    Ústrr!   )r%   r   Úcolumn_names      r   Ú_get_prompt_for_columnÚ'BaseDataCollator._get_prompt_for_columnE   s2   € ä�gœs×#Ñ#ØˆNÜ˜¤×&Ñ&¨;Ó+AØÑ'Ð'Ør   c                óÖ  • U(       d  0 $ [        US   R                  5       5      n0 nSU;   a  UR                  S5        US   S   US'   [        U5      U R                  ;  a  U R                  U5        U R                   HG  nXB;   d  M
  [        R                  " U Vs/ s H  oUU   PM	     sn5      US'   UR                  U5          O   U R                  U5      nU R                  U5      nU Hl  nUR                  US 5      n	U R                  Xx5      n
U Vs/ s H  oUU   PM	     nnU R                  XºU	S9nUR                  5        H  u  pÞXãU SU 3'   M     Mn     U$ s  snf s  snf )Nr   r   r   )ÚpromptÚtaskÚ_)r+   r,   ÚremoveÚtupler   Úmaybe_warn_about_column_orderr   ÚtorchÚtensorr'   r.   Úgetr3   r   Úitems)r%   ÚfeaturesÚcolumn_namesr&   Úlabel_columnÚrowr   r   r2   r7   r6   ÚinputsÚpreprocessedÚkeyÚvalues                  r   Ú__call__ÚBaseDataCollator.__call__M   ss  € ÞØˆIä˜H Q™K×,Ñ,Ó.Ó/ˆð ˆà˜\Ó)Ø×Ñ Ô/Ø$,¨Q¡K°Ñ$?ˆE�.Ñ!ä�Ó d×&:Ñ&:Ó:Ø×.Ñ.¨|Ô<ð !×4Ô4ˆLØÕ+Ü!&§¢ÉHÓ.UÊHÀS°<Ô/@ÉHÑ.UÓ!V��g‘Ø×#Ñ# LÔ1Ùñ	 5ð ×5Ñ5°eÓ<ˆØ×'Ñ'¨Ó.ˆã'ˆKØ!×%Ñ% k°4Ó8ˆDØ×0Ñ0°ÓFˆFÙ2:Ó;²(¨3˜+Ô&±(ˆFÐ;à×-Ñ-¨fÈ$Ð-ÐOˆLØ*×0Ñ0Ö2‘
�Ø05˜˜ Q s eÐ,Ó-ó 3ñ (ð ˆùò! /Vùò <s   ÂE!
ÄE&c                óŽ  • SSSSSSSSSSS.
nUR                  5        H€  u  p4X1;   d  M  UR                  U5      U:w  d  M#  US;   a  / SQnO US;   a  SS/nOUS	;   a  S
S/nO
US;   a  / SQn[        R                  SU< SUR                  U5       SU SW S3	5          O   U R                  R                  [        U5      5        g)zBWarn the user if the columns are likely not in the expected order.r   é   é   )
ÚanchorÚpositiveÚnegativeÚquestionÚanswerÚqueryÚresponseÚ
hypothesisÚ
entailmentÚcontradiction)rM   rN   rO   )rP   rQ   rP   rQ   )rR   rS   rR   rS   )rT   rU   rV   zColumn z is at index z?, whereas a column with this name is usually expected at index a?  . Note that the column order can be important for some losses, e.g. MultipleNegativesRankingLoss will always consider the first column as the anchor and the second as the positive, regardless of the dataset column names. Consider renaming the columns to match the expected order, e.g.:
dataset = dataset.select_columns(Ú)N)r?   ÚindexÚloggerÚwarningr   Úaddr:   )r%   rA   Úcolumn_name_to_expected_idxr2   Úexpected_idxÚproposed_fix_columnss         r   r;   Ú.BaseDataCollator.maybe_warn_about_column_orderr   s  € ð ØØØØØØØØØñ'
Ð#ð *E×)JÑ)JÖ)LÑ%ˆKØÕ*¨|×/AÑ/AÀ+Ó/NÐR^Õ/^ØÐ"DÓDÚ+MÑ(Ø Ð$:Ó:Ø,6¸Ð+AÑ(Ø Ð$9Ó9Ø,3°ZÐ+@Ñ(Ø Ð$QÓQÚ+XÐ(ä—‘Ø˜k™_¨M¸,×:LÑ:LÈ[Ó:YÐ9Zð [LØLXÈ>ð Z8ð 9MÐ7MÈQðPôñ ñ) *Mð, 	×Ñ× Ñ ¤ |Ó!4Õ5r   r   N)r&   údict[str, Any]Úreturnzdict[str, str])r&   r`   ra   zstr | dict[str, str] | None)r   zstr | dict[str, str]r2   r1   ra   z
str | None)r@   zlist[dict[str, Any]]ra   r`   )rA   r   ra   ÚNone)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__annotations__r   r   r!   r   r   Úsetr   r'   r.   r3   rH   r;   Ú__static_attributes__r   r   r   r	   r	      sw   ‡ ñ
ð ÓÙ%*Ñ;iÑ%jÐ˜ÓjÙHMÐ^bÐinÑHo€NÐEÓoÙGLÐ]aÐhmÑGn€GÐDÓná,1À#ÈEÐX]Ñ,^€OÐ)Ó^ô$ôô,ô#÷J%6r   r	   )Ú
__future__r   ÚloggingÚcollections.abcr   Údataclassesr   r   Útypingr   r<   Ú	getLoggerrc   rY   r	   r   r   r   Ú<module>rq      sC   ðÝ "ã Ý $ß (Ý ã à	×	Ò	˜8Ó	$€ð ÷I6ð I6ó ñI6r   