ó
    pyüi2  ã                   óÊ   • S SK r S SKrS SKrS SKJr  SSKJrJr  \R                  " \5      r	\ " S S5      5       r
\" SS9 " S	 S
5      5       r " S S5      r " S S\5      rg)é    N)Ú	dataclassé   )Úis_torch_availableÚloggingc                   ó\   • \ rS rSr% Sr\\S'   \\S'   Sr\S-  \S'   Sr\S-  \S'   S r	S	r
g)
ÚInputExampleé   a  
A single training/test example for simple sequence classification.

Args:
    guid: Unique id for the example.
    text_a: string. The untokenized text of the first sequence. For single
        sequence tasks, only this sequence must be specified.
    text_b: (Optional) string. The untokenized text of the second sequence.
        Only must be specified for sequence pair tasks.
    label: (Optional) string. The label of the example. This should be
        specified for train and dev examples, but not for test examples.
ÚguidÚtext_aNÚtext_bÚlabelc                 óZ   • [         R                  " [        R                  " U 5      SS9S-   $ )ú*Serializes this instance to a JSON string.é   )ÚindentÚ
©ÚjsonÚdumpsÚdataclassesÚasdict©Úselfs    Ú_/home/mande/repo/quber/.venv/lib/python3.13/site-packages/transformers/data/processors/utils.pyÚto_json_stringÚInputExample.to_json_string/   s#   € ä�zŠzœ+×,Ò,¨TÓ2¸1Ñ=ÀÑDÐDó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚstrÚ__annotations__r   r   r   Ú__static_attributes__r   r   r   r   r      s5   ‡ ñð ƒIØƒKØ€FˆC�$‰JÓØ€Eˆ3�‰:ÓõEr   r   T)Úfrozenc                   ó~   • \ rS rSr% Sr\\   \S'   Sr\\   S-  \S'   Sr	\\   S-  \S'   Sr
\\-  S-  \S'   S rS	rg)
ÚInputFeaturesé4   a“  
A single set of features of data. Property names are the same names as the corresponding inputs to a model.

Args:
    input_ids: Indices of input sequence tokens in the vocabulary.
    attention_mask: Mask to avoid performing attention on padding token indices.
        Mask values selected in `[0, 1]`: Usually `1` for tokens that are NOT MASKED, `0` for MASKED (padded)
        tokens.
    token_type_ids: (Optional) Segment token indices to indicate first and second
        portions of the inputs. Only some models use them.
    label: (Optional) Label corresponding to the input. Int for classification problems,
        float for regression problems.
Ú	input_idsNÚattention_maskÚtoken_type_idsr   c                 ó\   • [         R                  " [        R                  " U 5      5      S-   $ )r   r   r   r   s    r   r   ÚInputFeatures.to_json_stringI   s!   € ä�zŠzœ+×,Ò,¨TÓ2Ó3°dÑ:Ð:r   r   )r   r    r!   r"   r#   ÚlistÚintr%   r,   r-   r   Úfloatr   r&   r   r   r   r)   r)   4   sQ   ‡ ñð �C‰yÓØ'+€N�D˜‘I Ñ$Ó+Ø'+€N�D˜‘I Ñ$Ó+Ø $€Eˆ3�‰;˜ÑÓ$õ;r   r)   c                   óP   • \ rS rSrSrS rS rS rS rS r	S r
\SS
 j5       rSrg	)ÚDataProcessoréN   zEBase class for data converters for sequence classification data sets.c                 ó   • [        5       e)z�
Gets an example from a dict.

Args:
    tensor_dict: Keys and values should match the corresponding Glue
        tensorflow_dataset examples.
©ÚNotImplementedError)r   Útensor_dicts     r   Úget_example_from_tensor_dictÚ*DataProcessor.get_example_from_tensor_dictQ   s   € ô "Ó#Ð#r   c                 ó   • [        5       e)z8Gets a collection of [`InputExample`] for the train set.r7   ©r   Údata_dirs     r   Úget_train_examplesÚ DataProcessor.get_train_examples[   ó   € ä!Ó#Ð#r   c                 ó   • [        5       e)z6Gets a collection of [`InputExample`] for the dev set.r7   r=   s     r   Úget_dev_examplesÚDataProcessor.get_dev_examples_   rA   r   c                 ó   • [        5       e)z7Gets a collection of [`InputExample`] for the test set.r7   r=   s     r   Úget_test_examplesÚDataProcessor.get_test_examplesc   rA   r   c                 ó   • [        5       e)z*Gets the list of labels for this data set.r7   r   s    r   Ú
get_labelsÚDataProcessor.get_labelsg   rA   r   c                 ó–   • [        U R                  5       5      S:”  a+  U R                  5       [        UR                  5         Ul        U$ )zŽ
Some tensorflow_datasets datasets are not formatted the same way the GLUE datasets are. This method converts
examples to the correct format.
é   )ÚlenrI   r1   r   )r   Úexamples     r   Útfds_mapÚDataProcessor.tfds_mapk   s9   € ô
 ˆt�‰Ó Ó! AÓ%Ø ŸO™OÓ-¬c°'·-±-Ó.@ÑAˆGŒMØˆr   Nc           	      óŒ   • [        USSS9 n[        [        R                  " USUS95      sSSS5        $ ! , (       d  f       g= f)z!Reads a tab separated value file.Úrz	utf-8-sig)ÚencodingÚ	)Ú	delimiterÚ	quotecharN)Úopenr0   ÚcsvÚreader)ÚclsÚ
input_filerV   Úfs       r   Ú	_read_tsvÚDataProcessor._read_tsvt   s3   € ô �*˜c¨KÒ8¸AÜœŸ
š
 1°À	ÑJÓK÷ 9×8×8ús	   Œ5µ
Ar   ©N)r   r    r!   r"   r#   r:   r?   rC   rF   rI   rO   Úclassmethodr]   r&   r   r   r   r4   r4   N   s9   † ÙOò$ò$ò$ò$ò$òð óLó óLr   r4   c                   ó�   • \ rS rSrSrSS jrS rS r\ SS j5       r	\SS j5       r
       SS	 jr SS
 jr     SS jrSrg)Ú%SingleSentenceClassificationProcessoré{   z@Generic processor for a single sentence classification data set.Nc                 óL   • Uc  / OUU l         Uc  / OUU l        X0l        X@l        g r_   )ÚlabelsÚexamplesÚmodeÚverbose)r   re   rf   rg   rh   s        r   Ú__init__Ú.SingleSentenceClassificationProcessor.__init__~   s'   € Ø"™N‘b°ˆŒØ&Ñ.™°HˆŒØŒ	Ø�r   c                 ó,   • [        U R                  5      $ r_   )rM   rf   r   s    r   Ú__len__Ú-SingleSentenceClassificationProcessor.__len__„   s   € Ü�4—=‘=Ó!Ð!r   c                 óŒ   • [        U[        5      (       a!  [        U R                  U R                  U   S9$ U R                  U   $ )N)re   rf   )Ú
isinstanceÚslicerb   re   rf   )r   Úidxs     r   Ú__getitem__Ú1SingleSentenceClassificationProcessor.__getitem__‡   s<   € Ü�cœ5×!Ñ!Ü8ÀÇÁÐVZ×VcÑVcÐdgÑVhÑiÐiØ�}‰}˜SÑ!Ð!r   c                 óB   • U " S0 UD6nUR                  UUUUUUSSS9  U$ )NT)Ú
split_nameÚcolumn_labelÚcolumn_textÚ	column_idÚskip_first_rowÚoverwrite_labelsÚoverwrite_examplesr   )Úadd_examples_from_csv)	rZ   Ú	file_nameru   rv   rw   rx   ry   ÚkwargsÚ	processors	            r   Úcreate_from_csvÚ5SingleSentenceClassificationProcessor.create_from_csvŒ   sB   € ñ ‘M˜&‘Mˆ	Ø×'Ñ'ØØ!Ø%Ø#ØØ)Ø!Ø#ð 	(ñ 		
ð Ðr   c                 ó4   • U " S0 UD6nUR                  XS9  U$ )N)re   r   )Úadd_examples)rZ   Útexts_or_text_and_labelsre   r~   r   s        r   Úcreate_from_examplesÚ:SingleSentenceClassificationProcessor.create_from_examples�   s%   € á‘M˜&‘Mˆ	Ø×ÑÐ7ÐÑGØÐr   c	                 ób  • U R                  U5      n	U(       a  U	SS  n	/ n
/ n/ n[        U	5       Hm  u  pÞU
R                  Xä   5        UR                  Xã   5        Ub  UR                  Xå   5        MC  U(       a  U SU 3O
[        U5      nUR                  U5        Mo     U R	                  X«XÇUS9$ )NrL   Ú-)rz   r{   )r]   Ú	enumerateÚappendr$   rƒ   )r   r}   ru   rv   rw   rx   ry   rz   r{   ÚlinesÚtextsre   ÚidsÚiÚliner
   s                   r   r|   Ú;SingleSentenceClassificationProcessor.add_examples_from_csv£   s¸   € ð —‘˜yÓ)ˆÞØ˜!˜"�IˆEØˆØˆØˆÜ  Ö'‰GˆAØ�L‰L˜Ñ*Ô+Ø�M‰M˜$Ñ,Ô-ØÑ$Ø—
‘
˜4™?Ö+æ.8˜*˜ Q q cÑ*¼cÀ!»f�Ø—
‘
˜4Ö ñ (ð × Ñ Ø˜3ÐVhð !ð 
ð 	
r   c           
      ó:  • Ub;  [        U5      [        U5      :w  a#  [        S[        U5       S[        U5       35      eUb;  [        U5      [        U5      :w  a#  [        S[        U5       S[        U5       35      eUc  S /[        U5      -  nUc  S /[        U5      -  n/ n[        5       n[        XU5       HV  u  p‰n
[	        U[
        [        45      (       a  U	c  Uu  p¹OUnUR                  U	5        UR                  [        X«S U	S95        MX     U(       a  X`l
        OU R                  R                  U5        U(       a  [        U5      U l        U R                  $ [        [        U R                  5      R                  U5      5      U l        U R                  $ )Nz(Text and labels have mismatched lengths z and z%Text and ids have mismatched lengths )r
   r   r   r   )rM   Ú
ValueErrorÚsetÚzipro   Útupler0   ÚaddrŠ   r   rf   Úextendre   Úunion)r   r„   re   r�   rz   r{   rf   Úadded_labelsÚtext_or_text_and_labelr   r
   Útexts               r   rƒ   Ú2SingleSentenceClassificationProcessor.add_examplesÁ   sŠ  € ð Ñ¤#Ð&>Ó"?Ä3ÀvÃ;Ó"NÜØ:¼3Ð?WÓ;XÐ:YÐY^Ô_bÐciÓ_jÐ^kÐlóð ð ‰?œsÐ#;Ó<ÄÀCÃÓHÜÐDÄSÐIaÓEbÐDcÐchÔilÐmpÓiqÐhrÐsÓtÐtØ‰;Ø�&œ3Ð7Ó8Ñ8ˆCØ‰>Ø�VœcÐ":Ó;Ñ;ˆFØˆÜ“uˆÜ36Ð7OÐY\Ö3]Ñ/Ð"¨4ÜÐ0´5¼$°-×@Ñ@ÀUÁ]Ø4‘��eà-�Ø×Ñ˜UÔ#Ø�O‰OœL¨dÈÐTYÑZÖ[ñ 4^ö Ø$�Mà�M‰M× Ñ  Ô*ö Ü˜|Ó,ˆDŒKð �}‰}Ðô œs 4§;¡;Ó/×5Ñ5°lÓCÓDˆDŒKà�}‰}Ðr   c                 ó^	  • Uc  UR                   n[        U R                  5       VVs0 s H  u  pxX‡_M	     n	nn/ n
[        U R                  5       He  u  p¼US-  S:X  a  [        R                  SU 35        UR                  UR                  S[        X!R                   5      S9nU
R                  U5        Mg     [        S U
 5       5      n/ n[        [        X R                  5      5       GH_  u  nu  pÜUS-  S:X  a.  [        R                  SU S	[        U R                  5       35        U(       a  S
OS/[        U5      -  nU[        U5      -
  nU(       a  U/U-  U-   nU(       a  SOS
/U-  U-   nOXÔ/U-  -   nUU(       a  SOS
/U-  -   n[        U5      U:w  a  [        S[        U5       SU 35      e[        U5      U:w  a  [        S[        U5       SU 35      eU R                  S:X  a  XœR                     nO;U R                  S:X  a  [!        UR                  5      nO[        U R                  5      eUS:  að  U R"                  (       aß  [        R                  S5        [        R                  SUR$                   35        [        R                  SSR'                  U Vs/ s H  n[)        U5      PM     sn5       35        [        R                  SSR'                  U Vs/ s H  n[)        U5      PM     sn5       35        [        R                  SUR                   SU S35        UR                  [+        UUUS95        GMb     Uc  U$ US:X  Ga%  [-        5       (       d  [/        S5      eSSKnSSKJn  UR7                  U Vs/ s H  nUR8                  PM     snUR:                  S9n
UR7                  U Vs/ s H  nUR<                  PM     snUR:                  S9nU R                  S:X  a6  UR7                  U Vs/ s H  nUR                  PM     snUR:                  S9nOEU R                  S:X  a5  UR7                  U Vs/ s H  nUR                  PM     snUR                   S9nU" U
UW5      nU$ [        S5      es  snnf s  snf s  snf s  snf s  snf s  snf s  snf )au  
Convert examples in a list of `InputFeatures`

Args:
    tokenizer: Instance of a tokenizer that will tokenize the examples
    max_length: Maximum example length
    pad_on_left: If set to `True`, the examples will be padded on the left rather than on the right (default)
    pad_token: Padding token
    mask_padding_with_zero: If set to `True`, the attention mask will be filled by `1` for actual values
        and by `0` for padded values. If set to `False`, inverts it (`1` for padded values, `0` for actual
        values)

Returns:
    Will return a list of task-specific `InputFeatures` which can be fed to the model.

Ni'  r   zTokenizing example T)Úadd_special_tokensÚ
max_lengthc              3   ó8   #   • U  H  n[        U5      v •  M     g 7fr_   )rM   )Ú.0r+   s     r   Ú	<genexpr>ÚESingleSentenceClassificationProcessor.get_features.<locals>.<genexpr>  s   é € ÐIº=¨iœ3˜yŸ>˜>º=ùs   ‚zWriting example Ú/rL   zError with input length z vs ÚclassificationÚ
regressioné   z*** Example ***zguid: zinput_ids: Ú zattention_mask: zlabel: z (id = Ú))r+   r,   r   Úptz8return_tensors set to 'pt' but PyTorch can't be imported)ÚTensorDataset)Údtypez)return_tensors should be `'pt'` or `None`)Úmax_lenr‰   re   rf   ÚloggerÚinfoÚencoder   ÚminrŠ   Úmaxr”   rM   r’   rg   r   r2   rh   r
   Újoinr$   r)   r   ÚRuntimeErrorÚtorchÚtorch.utils.datar«   Útensorr+   Úlongr,   )r   Ú	tokenizerrŸ   Úpad_on_leftÚ	pad_tokenÚmask_padding_with_zeroÚreturn_tensorsrŽ   r   Ú	label_mapÚall_input_idsÚex_indexrN   r+   Úbatch_lengthÚfeaturesr,   Úpadding_lengthÚxrµ   r«   r\   Úall_attention_maskÚ
all_labelsÚdatasets                            r   Úget_featuresÚ2SingleSentenceClassificationProcessor.get_featuresæ   s/  € ð2 ÑØ"×*Ñ*ˆJä.7¸¿¹Ô.DÔEÒ.D¡( !�U’XÑ.Dˆ	ÑEàˆÜ!*¨4¯=©=Ö!9ÑˆHØ˜%Ñ 1Ó$Ü—‘Ð1°(°Ð<Ô=à!×(Ñ(Ø—‘Ø#'Ü˜z×+<Ñ+<Ó=ð )ð ˆIð
 × Ñ  Ö+ñ ":ô ÑI¹=ÓIÓIˆàˆÜ.7¼¸MÏ=É=Ó8Y×.ZÑ*ˆHÑ*�yØ˜%Ñ 1Ó$Ü—‘Ð.¨x¨j¸¼#¸d¿m¹mÓ:LÐ9MÐNÔOö $:™a¸qÐAÄCÈ	ÃNÑRˆNð *¬C°	«NÑ:ˆNÞØ'˜[¨>Ñ9¸YÑF�	Þ(>¡1ÀAÐ"FÈÑ"WÐ[iÑ!i‘à%¨°~Ñ)EÑF�	Ø!/Ö9O±AÐUVÐ3WÐZhÑ3hÑ!i�ä�9‹~ Ó-Ü Ð#;¼CÀ	»NÐ;KÈ4ÐP\È~Ð!^Ó_Ð_Ü�>Ó" lÓ2Ü Ð#;¼CÀÓ<OÐ;PÐPTÐUaÐTbÐ!cÓdÐdà�y‰yÐ,Ó,Ø!§-¡-Ñ0‘Ø—‘˜lÓ*Ü˜gŸm™mÓ,‘ä  §¡Ó+Ð+à˜!‹| §§Ü—‘Ð-Ô.Ü—‘˜f W§\¡\ NÐ3Ô4Ü—‘˜k¨#¯(©(ÁIÓ3NÂI¸q´C¸¶FÁIÑ3NÓ*OÐ)PÐQÔRÜ—‘Ð.¨s¯x©xÉÓ8XÊÀA¼¸Q¾ÉÑ8XÓ/YÐ.ZÐ[Ô\Ü—‘˜g g§m¡m _°G¸E¸7À!ÐDÔEà�O‰OœM°IÈnÐdiÑj×kñG /[ðJ Ñ!ØˆOØ˜tÔ#Ü%×'Ñ'Ü"Ð#]Ó^Ð^ÛÝ6à!ŸL™L¹xÓ)Hºx¸!¨!¯+¬+¹xÑ)HÐPU×PZÑPZ˜LÐ[ˆMØ!&§¡ÉÓ.RÊÀA¨q×/?Ô/?ÉÑ.RÐZ_×ZdÑZd Ð!eÐØ�y‰yÐ,Ó,Ø"Ÿ\™\¹HÓ*EºH°q¨1¯7¬7¹HÑ*EÈUÏZÉZ˜\ÐX‘
Ø—‘˜lÓ*Ø"Ÿ\™\¹HÓ*EºH°q¨1¯7¬7¹HÑ*EÈUÏ[É[˜\ÐY�
á# MÐ3EÀzÓRˆGØˆNäÐHÓIÐIùóQ Fùò` 4OùÚ8Xùò *IùÚ.Rùâ*Eùâ*Es)   ¨RÊ'RË(RÎRÏR ÐR%ÑR*)rf   re   rg   rh   )NNr¥   F)Ú r   rL   NFr_   )rÊ   r   rL   NFFF)NNFF)NFr   TN)r   r    r!   r"   r#   ri   rl   rr   r`   r€   r…   r|   rƒ   rÈ   r&   r   r   r   rb   rb   {   s€   † ÙJôò"ò"ð
 àejóó ðð  óó ðð ØØØØØØ ô
ð> kpô#ðP ØØØ#Ø÷dJr   rb   )rX   r   r   r   Úutilsr   r   Ú
get_loggerr   r®   r   r)   r4   rb   r   r   r   Ú<module>rÍ      s   ðó  Û Û Ý !ç 0ð 
×	Ò	˜HÓ	%€ð ÷Eð Eó ðEñ0 �$Ñ÷;ð ;ó ð;÷2*Lñ *LôZOJ¨Mõ OJr   