ó
    pyüißp  ã                   ó¤  • S SK r S SKrS SKJr  S SKJrJr  S SKJr  S SK	r
S SKJr  SSKJr  SSKJrJrJr  SS	KJrJrJr  S
SKJr  1 Skr\" 5       (       a
  S SKrS SKJr  \R6                  " \5      rS rS rS r S r!S r"S\4S jr#    S"S jr$ " S S\5      r% " S S\%5      r& " S S\%5      r' " S S5      r( " S S5      r) " S  S!5      r*g)#é    N)Úpartial)ÚPoolÚ	cpu_count)Ú
ThreadPool)Útqdmé   )Úwhitespace_tokenize)ÚBatchEncodingÚPreTrainedTokenizerBaseÚTruncationStrategy)Úis_torch_availableÚis_torch_hpu_availableÚloggingé   )ÚDataProcessor>   ÚbartÚmpnetÚrobertaÚ	camembert)ÚTensorDatasetc                 óâ   • SR                  UR                  U5      5      n[        XS-   5       H;  n[        X&S-
  S5       H%  nSR                  XUS-    5      nX…:X  d  M   Xg4s  s  $    M=     X4$ )zFReturns tokenized answer spans that better match the annotated answer.Ú r   éÿÿÿÿ)ÚjoinÚtokenizeÚrange)	Ú
doc_tokensÚinput_startÚ	input_endÚ	tokenizerÚorig_answer_textÚtok_answer_textÚ	new_startÚnew_endÚ	text_spans	            Ú_/home/mande/repo/quber/.venv/lib/python3.13/site-packages/transformers/data/processors/squad.pyÚ_improve_answer_spanr'   *   su   € à—h‘h˜y×1Ñ1Ð2BÓCÓD€Oä˜;°A©Ö6ˆ	Ü˜Y°A©°rÖ:ˆGØŸ™ ¸À1¹Ð!FÓGˆIØÕ+Ø!Ð+Ô+ó ;ñ 7ð Ð#Ð#ó    c                 ó  • SnSn[        U 5       Ht  u  pVUR                  UR                  -   S-
  nX&R                  :  a  M2  X':”  a  M9  X&R                  -
  nXr-
  n	[        X‰5      SUR                  -  -   n
Ub  X£:”  d  Mp  U
nUnMv     X:H  $ )ú:Check if this is the 'max context' doc span for the token.Nr   ç{®Gáz„?)Ú	enumerateÚstartÚlengthÚmin©Ú	doc_spansÚcur_span_indexÚpositionÚ
best_scoreÚbest_span_indexÚ
span_indexÚdoc_spanÚendÚnum_left_contextÚnum_right_contextÚscores              r&   Ú_check_is_max_contextr<   7   s˜   € à€JØ€OÜ )¨)Ö 4Ñˆ
Ø�n‰n˜xŸ™Ñ.°Ñ2ˆØ—n‘nÓ$ÙØ‹>ÙØ#§n¡nÑ4ÐØ™NÐÜÐ$Ó8¸4À(Ç/Á/Ñ;QÑQˆØÑ Õ!3ØˆJØ(ŠOñ !5ð Ñ,Ð,r(   c                 óÒ   • SnSn[        U 5       HQ  u  pVUS   US   -   S-
  nX&S   :  a  M  X':”  a  M$  X&S   -
  nXr-
  n	[        X‰5      SUS   -  -   n
Ub  X£:”  d  MM  U
nUnMS     X:H  $ )r*   Nr-   r.   r   r+   )r,   r/   r0   s              r&   Ú_new_check_is_max_contextr>   K   sŸ   € ð €JØ€OÜ )¨)Ö 4Ñˆ
Ø�wÑ (¨8Ñ"4Ñ4°qÑ8ˆØ˜wÑ'Ó'ÙØ‹>ÙØ#¨wÑ&7Ñ7ÐØ™NÐÜÐ$Ó8¸4À(È8ÑBTÑ;TÑTˆØÑ Õ!3ØˆJØ(ŠOñ !5ð Ñ,Ð,r(   c                 óT   • U S:X  d!  U S:X  d  U S:X  d  U S:X  d  [        U 5      S:X  a  gg)Nr   Ú	ÚÚ
i/   TF)Úord)Úcs    r&   Ú_is_whitespacerE   a   s,   € ØˆCƒx�1˜“9  T£	¨Q°$«Y¼#¸a»&ÀFÓ:JØØr(   c                 óø  • / nU(       a   U R                   (       d�  U R                  nU R                  nSR                  U R                  XxS-    5      n	SR                  [        U R                  5      5      n
U	R                  U
5      S:X  a  [        R                  SU	 SU
 S35        / $ / n/ n/ n[        U R                  5       H’  u  pïUR                  [        U5      5        [        R                  R                  S;   a  [        R!                  USS	9nO[        R!                  U5      nU H%  nUR                  U5        UR                  U5        M'     M”     U(       aˆ  U R                   (       dw  XÀR                     nU R                  [        U R                  5      S-
  :  a  XÀR                  S-      S-
  nO[        U5      S-
  n[#        UUU[        U R                  5      u  nn/ n[        R%                  U R&                  S
SUS9n[)        [        5      R                  R+                  SS5      R-                  5       nU[.        ;   a$  [        R0                  [        R2                  -
  S-   O [        R0                  [        R2                  -
  n[        R0                  [        R5                  SS9-
  n[        R0                  U-
  nUn[        U5      U-  [        U5      :  Ga9  [        R6                  S:X  a  UnUn[8        R:                  R<                  nOUnUn[8        R>                  R<                  n[        UUUUUSX-
  [        U5      -
  U-
  SS9n[A        [        U5      [        U5      U-  -
  U[        U5      -
  U-
  5      n[        RB                  US   ;   aƒ  [        R6                  S:X  a)  US   S US   RE                  [        RB                  5       n OK[        US   5      S-
  US   S S S2   RE                  [        RB                  5      -
  n!US   U!S-   S  n OUS   n [        RG                  U 5      n"0 n#[I        U5       HA  n[        R6                  S:X  a  [        U5      U-   U-   OUn$U[        U5      U-  U-      U#U$'   MC     UUS'   U"US'   U#US'   [        U5      U-   US'   0 US'   [        U5      U-  US'   UUS'   UR                  U5        SU;  d  SU;   a  [        US   5      S:X  a  O#US   n[        U5      U-  [        U5      :  a  GM9  [I        [        U5      5       HZ  n%[I        UU%   S   5       HB  n&[K        UU%U%U-  U&-   5      n'[        R6                  S:X  a  U&O
UU%   S   U&-   n$U'UU%   S   U$'   MD     M\     U GHá  n(U(S   RE                  [        RL                  5      n)[N        RP                  " U(S   5      n*[        R6                  S:X  a  SU*[        U5      U-   S & OSU*[        U(S   5      * [        U5      U-   * & [N        RR                  " [N        RT                  " U(S   [        RB                  :H  5      5      n+[N        RV                  " [        RY                  U(S   SS95      R[                  5       n,SU*U+'   SU*U,'   SU*U)'   U R                   n-SnSnU(       am  U-(       df  U(S   n.U(S   U(S   -   S-
  n/S
n0WU.:¼  a  WU/::  d  Sn0U0(       a  U)nU)nSn-O5[        R6                  S:X  a  Sn1O[        U5      U-   n1UU.-
  U1-   nWU.-
  U1-   nUR                  []        U(S   U(S   U(S   U)U*R_                  5       SSU(S   U(S   U(S   U(S   UUU-U R`                  S95        GMä     U$ ) Nr   r   r   zCould not find answer: 'z' vs. 'Ú')ÚRobertaTokenizerÚLongformerTokenizerÚBartTokenizerÚLongformerTokenizerFastÚBartTokenizerFastT)Úadd_prefix_spaceF)Úadd_special_tokensÚ
truncationÚ
max_lengthÚ	TokenizerÚ )ÚpairÚright)rO   ÚpaddingrP   Úreturn_overflowing_tokensÚstrideÚreturn_token_type_idsÚ	input_idsÚparagraph_lenÚtokensÚtoken_to_orig_mapÚ*truncated_query_with_special_tokens_lengthÚtoken_is_max_contextr-   r.   Úoverflowing_tokensr   ÚleftÚtoken_type_ids)Úalready_has_special_tokensÚattention_mask)
Úexample_indexÚ	unique_idrZ   r^   r[   r\   Ústart_positionÚend_positionÚis_impossibleÚqas_id)1rh   rf   rg   r   r   r	   Úanswer_textÚfindÚloggerÚwarningr,   ÚappendÚlenr    Ú	__class__Ú__name__r   r'   ÚencodeÚquestion_textÚtypeÚreplaceÚlowerÚMULTI_SEP_TOKENS_TOKENIZERS_SETÚmodel_max_lengthÚmax_len_single_sentenceÚnum_special_tokens_to_addÚpadding_sider   ÚONLY_SECONDÚvalueÚ
ONLY_FIRSTr/   Úpad_token_idÚindexÚconvert_ids_to_tokensr   r>   Úcls_token_idÚnpÚ	ones_likeÚwhereÚ
atleast_1dÚasarrayÚget_special_tokens_maskÚnonzeroÚSquadFeaturesÚtolistri   )2ÚexampleÚmax_seq_lengthÚ
doc_strideÚmax_query_lengthÚpadding_strategyÚis_trainingÚfeaturesrf   rg   Úactual_textÚcleaned_answer_textÚtok_to_orig_indexÚorig_to_tok_indexÚall_doc_tokensÚiÚtokenÚ
sub_tokensÚ	sub_tokenÚtok_start_positionÚtok_end_positionÚspansÚtruncated_queryÚtokenizer_typeÚsequence_added_tokensÚmax_len_sentences_pairÚsequence_pair_added_tokensÚspan_doc_tokensÚtextsÚpairsrO   Úencoded_dictrZ   Únon_padded_idsÚlast_padding_id_positionr[   r\   r€   Údoc_span_indexÚjÚis_max_contextÚspanÚ	cls_indexÚp_maskÚpad_token_indicesÚspecial_token_indicesÚspan_is_impossibleÚ	doc_startÚdoc_endÚout_of_spanÚ
doc_offsets2                                                     r&   Ú!squad_convert_example_to_featuresr·   g   sä  € ð €HÞ˜7×0×0à ×/Ñ/ˆØ×+Ñ+ˆð —h‘h˜w×1Ñ1°.ÐSTÑDTÐVÓWˆØ!Ÿh™hÔ':¸7×;NÑ;NÓ'OÓPÐØ×ÑÐ/Ó0°BÓ6Ü�N‰NÐ5°k°]À'ÐJ]ÐI^Ð^_Ð`ÔaØˆIàÐØÐØ€NÜ˜g×0Ñ0Ö1‰ˆØ× Ñ ¤ ^Ó!4Ô5Ü×Ñ×'Ñ'ð ,
ó 
ô #×+Ñ+¨EÀDÐ+ÐI‰Jä"×+Ñ+¨EÓ2ˆJÛ#ˆIØ×$Ñ$ QÔ'Ø×!Ñ! )Ö,ó $ñ 2ö  ˜7×0×0Ø.×/EÑ/EÑFÐØ×Ñ¤# g×&8Ñ&8Ó"9¸AÑ"=Ó=Ø0×1EÑ1EÈÑ1IÑJÈQÑNÑä" >Ó2°QÑ6Ðä1EØÐ.Ð0@Ä)ÈW×M`ÑM`ó2
Ñ.Ð	Ð-ð €Eä×&Ñ&Ø×Ñ°%ÀDÐUeð 'ð €Oô œ)“_×-Ñ-×5Ñ5°kÀ2ÓF×LÑLÓN€Nð Ô<Ó<ô 	×"Ñ"¤Y×%FÑ%FÑFÈÒJä×'Ñ'¬)×*KÑ*KÑKð ô
 '×7Ñ7¼)×:]Ñ:]ÐcgÐ:]Ð:hÑhÐÜ!*×!;Ñ!;Ð>TÑ!TÐà$€OÜ
ˆe‹*�zÑ
!¤C¨Ó$7Ô
7ä×!Ñ! WÓ,Ø#ˆEØ#ˆEÜ+×7Ñ7×=Ñ=‰Jà#ˆEØ#ˆEÜ+×6Ñ6×<Ñ<ˆJä ØØØ!Ø$Ø%Ø&*Ø!Ñ.´°_Ó1EÑEÐHbÑbØ"&ñ	
ˆô Ü�Ó¤# e£*¨zÑ"9Ñ9ØœS Ó1Ñ1Ð4NÑNó
ˆô
 ×!Ñ! \°+Ñ%>Ó>Ü×%Ñ%¨Ó0Ø!-¨kÑ!:Ð;t¸\È+Ñ=V×=\Ñ=\Ô]f×]sÑ]sÓ=tÐ!u‘ô ˜ [Ñ1Ó2°QÑ6¸ÀkÑ9RÑSWÐUWÐSWÑ9X×9^Ñ9^Ô_h×_uÑ_uÓ9vÑvð )ð ".¨kÑ!:Ð;SÐVWÑ;WÐ;YÐ!Z‘ð *¨+Ñ6ˆNä×0Ñ0°Ó@ˆàÐÜ�}Ö%ˆAÜHQ×H^ÑH^ÐbiÓHi”C˜Ó(Ð+@Ñ@À1ÒDÐopˆEØ'8¼¸U»ÀjÑ9PÐSTÑ9TÑ'UÐ˜eÓ$ñ &ð )6ˆ�_Ñ%Ø!'ˆ�XÑØ,=ˆÐ(Ñ)ÜEHÈÓEYÐ\qÑEqˆÐAÑBØ/1ˆÐ+Ñ,Ü # E£
¨ZÑ 7ˆ�WÑØ!.ˆ�XÑà�‰�\Ô"à |Ó3Ø  LÓ0´S¸ÐFZÑ9[Ó5\Ð`aÓ5aàØ&Ð';Ñ<ˆôy ˆe‹*�zÑ
!¤C¨Ó$7Ö
7ô|  ¤ E£
Ö+ˆÜ�u˜^Ñ,¨_Ñ=Ö>ˆAÜ6°u¸nÈnÐ_iÑNiÐlmÑNmÓnˆNô ×)Ñ)¨VÓ3ñ à˜>Ñ*Ð+WÑXÐ[\Ñ\ð ð
 DRˆE�.Ñ!Ð"8Ñ9¸%Ó@ó ?ñ ,ô ˆà˜Ñ%×+Ñ+¬I×,BÑ,BÓCˆ	ô —’˜dÐ#3Ñ4Ó5ˆÜ×!Ñ! WÓ,ØEFˆF”3�Ó'Ð*?Ñ?ÐAÑBà]^ˆF”C˜˜X™Ó'Ð'¬C°Ó,@ÐCXÑ,XÐ*YÐZäŸHšH¤R§]¢]°4¸Ñ3DÌ	×H^ÑH^Ñ3^Ó%_Ó`ÐÜ "§
¢
Ü×-Ñ-¨d°;Ñ.?Ð\`Ð-Ðaó!
ç
‰'‹)ð 	ð %&ˆÐ Ñ!Ø()ˆÐ$Ñ%ð ˆˆyÑà$×2Ñ2ÐØˆØˆÞÖ1ð ˜W™ˆIØ˜7‘m d¨8¡nÑ4°qÑ8ˆGØˆKà&¨)Ó3Ð8HÈGÓ8SØ"�æØ!*�Ø(�Ø%)Ñ"ä×)Ñ)¨VÓ3Ø!"‘Jä!$ _Ó!5Ð8MÑ!M�Jà!3°iÑ!?À*Ñ!L�Ø/°)Ñ;¸jÑH�Ø�‰ÜØ�[Ñ!ØÐ%Ñ&ØÐ%Ñ&ØØ—‘“ØØØ" ?Ñ3Ø%)Ð*@Ñ%AØ˜H‘~Ø"&Ð':Ñ";Ø-Ø)Ø0Ø—~‘~ñ÷	
ñ_ ðD €Or(   Útokenizer_for_convertc                 ó   • U q g ©N)r    )r¸   s    r&   Ú&squad_convert_example_to_features_initr»   4  s   € à%�Ir(   c
                 ó<  • [        U[        5       5      n[        5       (       a  [        O[        n
U
" U[
        U4S9 n[        [        UUUUUS9n[        [        UR                  XÀSS9[        U 5      SU	(       + S95      nSSS5        / nSnS	n[        W[        U5      S
U	(       + S9 H>  nU(       d  M  U H'  nUUl        UUl        UR                  U5        US-  nM)     US-  nM@     UnAUS:X  Gam  [        5       (       d  [!        S5      e["        R$                  " U Vs/ s H  nUR&                  PM     sn["        R(                  S9n["        R$                  " U Vs/ s H  nUR*                  PM     sn["        R(                  S9n["        R$                  " U Vs/ s H  nUR,                  PM     sn["        R(                  S9n["        R$                  " U Vs/ s H  nUR.                  PM     sn["        R(                  S9n["        R$                  " U Vs/ s H  nUR0                  PM     sn["        R2                  S9n["        R$                  " U Vs/ s H  nUR4                  PM     sn["        R2                  S9nU(       dF  ["        R6                  " UR9                  S	5      ["        R(                  S9n[;        UUUUUU5      nUU4$ ["        R$                  " U Vs/ s H  nUR<                  PM     sn["        R(                  S9n["        R$                  " U Vs/ s H  nUR>                  PM     sn["        R(                  S9n[;        UUUUUUUU5      nUU4$ U$ ! , (       d  f       GNê= fs  snf s  snf s  snf s  snf s  snf s  snf s  snf s  snf )a  
Converts a list of examples into a list of features that can be directly given as input to a model. It is
model-dependant and takes advantage of many of the tokenizer's features to create the model's inputs.

Args:
    examples: list of [`~data.processors.squad.SquadExample`]
    tokenizer: an instance of a child of [`PreTrainedTokenizer`]
    max_seq_length: The maximum sequence length of the inputs.
    doc_stride: The stride used when the context is too large and is split across several features.
    max_query_length: The maximum length of the query.
    is_training: whether to create features for model evaluation or model training.
    padding_strategy: Default to "max_length". Which padding strategy to use
    return_dataset: Default False. Can also be 'pt'.
        if 'pt': returns a torch.data.TensorDataset.
    threads: multiple processing threads.


Returns:
    list of [`~data.processors.squad.SquadFeatures`]

Example:

```python
processor = SquadV2Processor()
examples = processor.get_dev_examples(data_dir)

features = squad_convert_examples_to_features(
    examples=examples,
    tokenizer=tokenizer,
    max_seq_length=args.max_seq_length,
    doc_stride=args.doc_stride,
    max_query_length=args.max_query_length,
    is_training=not evaluate,
)
```)ÚinitializerÚinitargs)r�   rŽ   r�   r�   r‘   é    )Ú	chunksizez"convert squad examples to features)ÚtotalÚdescÚdisableNi Êš;r   zadd example index and unique idr   Úptz6PyTorch must be installed to return a PyTorch dataset.)Údtype) r/   r   r   r   r   r»   r   r·   Úlistr   Úimapro   rd   re   rn   r   ÚRuntimeErrorÚtorchÚtensorrY   Úlongrc   ra   r®   r¯   Úfloatrh   ÚarangeÚsizer   rf   rg   )Úexamplesr    r�   rŽ   r�   r‘   r�   Úreturn_datasetÚthreadsÚtqdm_enabledÚpool_clsÚpÚ	annotate_r’   Únew_featuresre   rd   Úexample_featuresÚexample_featureÚfÚall_input_idsÚall_attention_masksÚall_token_type_idsÚall_cls_indexÚ
all_p_maskÚall_is_impossibleÚall_feature_indexÚdatasetÚall_start_positionsÚall_end_positionss                                 r&   Ú"squad_convert_examples_to_featuresrä   9  s*  € ô` �'œ9›;Ó'€GÜ3×5Ñ5�z¼4€HÙ	�'Ô'MÐYbÐXdÒ	eÐijÜÜ-Ø)Ø!Ø-Ø-Ø#ñ
ˆ	ô ÜØ—‘�y°b�Ð9Ü˜(“mØ9Ø(Ô(ñ	ó
ˆ÷ 
fð$ €LØ€IØ€MÜ Øœ˜H›Ð,MÐ[gÔWgôÐö  ÙÛ/ˆOØ,9ˆOÔ)Ø(1ˆOÔ%Ø×Ñ Ô0Ø˜‰NŠIñ	  0ð
 	˜ÑŠñð €HØØ˜ÔÜ!×#Ñ#ÜÐWÓXÐXô Ÿš¹8Ó%Dº8°a a§k¤k¹8Ñ%DÌEÏJÉJÑWˆÜ#ŸlšlÁhÓ+OÂhÀ¨A×,<Ô,<ÁhÑ+OÔW\×WaÑWaÑbÐÜ"Ÿ\š\ÁXÓ*NÂXÀ¨1×+;Ô+;ÁXÑ*NÔV[×V`ÑV`ÑaÐÜŸš¹8Ó%Dº8°a a§k¤k¹8Ñ%DÌEÏJÉJÑWˆÜ—\’\±XÓ">²X° 1§8¤8±XÑ">ÄeÇkÁkÑRˆ
Ü!ŸLšLÁ8Ó)LÂ8¸a¨!¯/¬/Á8Ñ)LÔTY×T_ÑT_Ñ`ÐæÜ %§¢¨]×-?Ñ-?ÀÓ-BÌ%Ï*É*Ñ UÐÜ#ØÐ2Ð4FÐHYÐ[hÐjtóˆGð" ˜Ð Ð ô #(§,¢,É(Ó/SÊ(ÀQ°×0@Ô0@É(Ñ/SÔ[`×[eÑ[eÑ"fÐÜ %§¢ÁhÓ-OÂhÀ¨a¯n¬nÁhÑ-OÔW\×WaÑWaÑ bÐÜ#ØØ#Ø"Ø#Ø!ØØØ!ó	ˆGð ˜Ð Ð àˆ÷G 
fÖ	eüòN &EùÚ+OùÚ*NùÚ%DùÚ">ùÚ)Lùò 0TùÚ-Os<   ½AM$Ä#M6Å!M;ÆN ÇNÈN
ÉNË$NÌ"NÍ$
M3c                   óN   • \ rS rSrSrSrSrS
S jrS
S jrSS jr	SS jr
S rS	rg)ÚSquadProcessori±  z˜
Processor for the SQuAD data set. overridden by SquadV1Processor and SquadV2Processor, used by the version 1.1 and
version 2.0 of SQuAD, respectively.
Nc           
      ó~  • U(       dD  US   S   S   R                  5       R                  S5      nUS   S   S   R                  5       n/ nO^[        US   S   US   S   5       VVs/ s H5  u  pgUR                  5       UR                  5       R                  S5      S.PM7     nnnS nS n[        US   R                  5       R                  S5      US   R                  5       R                  S5      US	   R                  5       R                  S5      UUUS
   R                  5       R                  S5      US9$ s  snnf )NÚanswersÚtextr   úutf-8Úanswer_start)rë   ré   ÚidÚquestionÚcontextÚtitle)ri   rs   Úcontext_textrj   Ústart_position_characterrï   rè   )ÚnumpyÚdecodeÚzipÚSquadExample)ÚselfÚtensor_dictÚevaluateÚanswerrë   rè   r-   ré   s           r&   Ú_get_example_from_tensor_dictÚ,SquadProcessor._get_example_from_tensor_dictº  sC  € ÞØ  Ñ+¨FÑ3°AÑ6×<Ñ<Ó>×EÑEÀgÓNˆFØ& yÑ1°.ÑAÀ!ÑD×JÑJÓLˆLØ‰Gô $' {°9Ñ'=¸nÑ'MÈ{Ð[dÑOeÐflÑOmÔ#nôâ#n‘K�Eð "'§¡£¸¿
¹
»×8KÑ8KÈGÓ8TÔUÙ#nð ñ ð
 ˆFØˆLäØ˜tÑ$×*Ñ*Ó,×3Ñ3°GÓ<Ø% jÑ1×7Ñ7Ó9×@Ñ@ÀÓIØ$ YÑ/×5Ñ5Ó7×>Ñ>¸wÓGØØ%1Ø˜gÑ&×,Ñ,Ó.×5Ñ5°gÓ>Øñ
ð 	
ùós   Á'<D9c                 óŽ   • U(       a  US   nOUS   n/ n[        U5       H!  nUR                  U R                  XBS95        M#     U$ )a  
Creates a list of [`~data.processors.squad.SquadExample`] using a TFDS dataset.

Args:
    dataset: The tfds dataset loaded from *tensorflow_datasets.load("squad")*
    evaluate: Boolean specifying if in evaluation mode or in training mode

Returns:
    List of SquadExample

Examples:

```python
>>> import tensorflow_datasets as tfds

>>> dataset = tfds.load("squad")

>>> training_examples = get_examples_from_dataset(dataset, evaluate=False)
>>> evaluation_examples = get_examples_from_dataset(dataset, evaluate=True)
```Ú
validationÚtrain)rø   )r   rn   rú   )rö   rá   rø   rÏ   r÷   s        r&   Úget_examples_from_datasetÚ(SquadProcessor.get_examples_from_datasetÒ  sO   € ö, Ø˜lÑ+‰Gà˜gÑ&ˆGàˆÜ ž=ˆKØ�O‰O˜D×>Ñ>¸{Ð>Ð^Ö_ñ )ð ˆr(   c                 ó4  • Uc  SnU R                   c  [        S5      e[        [        R                  R                  Xc  U R                   OU5      SSS9 n[        R                  " U5      S   nSSS5        U R                  WS5      $ ! , (       d  f       N = f)	af  
Returns the training examples from the data directory.

Args:
    data_dir: Directory containing the data files used for training and evaluating.
    filename: None by default, specify this if the training file has a different name than the original one
        which is `train-v1.1.json` and `train-v2.0.json` for squad versions 1.1 and 2.0 respectively.

NrR   úNSquadProcessor should be instantiated via SquadV1Processor or SquadV2ProcessorÚrrê   ©ÚencodingÚdatarþ   )	Ú
train_fileÚ
ValueErrorÚopenÚosÚpathr   ÚjsonÚloadÚ_create_examples©rö   Údata_dirÚfilenameÚreaderÚ
input_datas        r&   Úget_train_examplesÚ!SquadProcessor.get_train_examplesó  s‰   € ð ÑØˆHà�?‰?Ñ"ÜÐmÓnÐnäÜ�G‰G�L‰L˜Ñ6F 4§?¢?ÈHÓUÐWZÐelò
àÜŸš 6Ó*¨6Ñ2ˆJ÷
ð ×$Ñ$ Z°Ó9Ð9÷	
õ 
úó   ÁB	Â	
Bc                 ó4  • Uc  SnU R                   c  [        S5      e[        [        R                  R                  Xc  U R                   OU5      SSS9 n[        R                  " U5      S   nSSS5        U R                  WS5      $ ! , (       d  f       N = f)	ad  
Returns the evaluation example from the data directory.

Args:
    data_dir: Directory containing the data files used for training and evaluating.
    filename: None by default, specify this if the evaluation file has a different name than the original one
        which is `dev-v1.1.json` and `dev-v2.0.json` for squad versions 1.1 and 2.0 respectively.
NrR   r  r  rê   r  r  Údev)	Údev_filer  r	  r
  r  r   r  r  r  r  s        r&   Úget_dev_examplesÚSquadProcessor.get_dev_examples	  s‰   € ð ÑØˆHà�=‰=Ñ ÜÐmÓnÐnäÜ�G‰G�L‰L˜Ñ4D 4§=¢=È(ÓSÐUXÐcjò
àÜŸš 6Ó*¨6Ñ2ˆJ÷
ð ×$Ñ$ Z°Ó7Ð7÷	
õ 
úr  c                 óN  • US:H  n/ n[        U5       HŽ  nUS   nUS    H}  nUS   nUS    Hl  n	U	S   n
U	S   nS nS n/ nU	R                  SS	5      nU(       d  U(       a  U	S
   S   nUS   nUS   nOU	S
   n[        U
UUUUUUUS9nUR                  U5        Mn     M     M�     U$ )Nrþ   rï   Ú
paragraphsrî   Úqasrì   rí   rh   Frè   r   ré   rë   )ri   rs   rð   rj   rñ   rï   rh   rè   )r   Úgetrõ   rn   )rö   r  Úset_typer‘   rÏ   Úentryrï   Ú	paragraphrð   Úqari   rs   rñ   rj   rè   rh   rù   rŒ   s                     r&   r  ÚSquadProcessor._create_examples  së   € Ø 'Ñ)ˆØˆÜ˜*Ö%ˆEØ˜'‘NˆEØ" <Ô0�	Ø(¨Ñ3�Ø# EÔ*�BØ ™X�FØ$& z¡N�MØ/3Ð,Ø"&�KØ �Gà$&§F¡F¨?¸EÓ$B�MÞ(Þ&Ø%'¨	¡]°1Ñ%5˜FØ*0°©.˜KØ7=¸nÑ7MÑ4à&(¨¡m˜Gä*Ø%Ø&3Ø%1Ø$/Ø1IØ#Ø&3Ø 'ñ	�Gð —O‘O GÖ,ó5 +ó 1ñ &ð> ˆr(   © )Frº   )rq   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r  r  rú   rÿ   r  r  r  Ú__static_attributes__r%  r(   r&   ræ   ræ   ±  s-   † ñð
 €JØ€Hô
ô0ôB:ô,8õ*"r(   ræ   c                   ó   • \ rS rSrSrSrSrg)ÚSquadV1ProcessoriC  ztrain-v1.1.jsonzdev-v1.1.jsonr%  N©rq   r&  r'  r(  r  r  r*  r%  r(   r&   r,  r,  C  ó   † Ø"€JØƒHr(   r,  c                   ó   • \ rS rSrSrSrSrg)ÚSquadV2ProcessoriH  ztrain-v2.0.jsonzdev-v2.0.jsonr%  Nr-  r%  r(   r&   r0  r0  H  r.  r(   r0  c                   ó&   • \ rS rSrSr/ S4S jrSrg)rõ   iM  a(  
A single training/test example for the Squad dataset, as loaded from disk.

Args:
    qas_id: The example's unique identifier
    question_text: The question string
    context_text: The context string
    answer_text: The answer string
    start_position_character: The character position of the start of the answer
    title: The title of the example
    answers: None by default, this is used during evaluation. Holds answers as well as their start positions.
    is_impossible: False by default, set to True if the example has no possible answer.
Fc	                 óø  • Xl         X l        X0l        X@l        X`l        X€l        Xpl        Su  U l        U l        / n	/ n
SnU R                   H[  n[        U5      (       a  SnO(U(       a  U	R                  U5        OU	S==   U-  ss'   SnU
R                  [        U	5      S-
  5        M]     X�l        X l        UbA  U(       d9  X¥   U l        U
[        U[        U5      -   S-
  [        U
5      S-
  5         U l        g g g )N)r   r   Tr   Fr   )ri   rs   rð   rj   rï   rh   rè   rf   rg   rE   rn   ro   r   Úchar_to_word_offsetr/   )rö   ri   rs   rð   rj   rñ   rï   rè   rh   r   r3  Úprev_is_whitespacerD   s                r&   Ú__init__ÚSquadExample.__init__\  s	  € ð ŒØ*ÔØ(ÔØ&ÔØŒ
Ø*ÔØŒà15Ñ.ˆÔ˜TÔ.àˆ
Ø ÐØ!Ðð ×"Ô"ˆAÜ˜a× Ñ Ø%)Ñ"æ%Ø×%Ñ% aÕ(à˜r“N aÑ'“NØ%*Ð"Ø×&Ñ&¤s¨:£¸Ñ':Ö;ñ #ð %ŒØ#6Ô ð $Ñ/¾Ø"5Ñ"OˆDÔØ 3ÜÐ,¬s°;Ó/?Ñ?À!ÑCÄSÐI\ÓE]Ð`aÑEaÓbñ!ˆDÕð 9FÐ/r(   )rj   rè   r3  rð   r   rg   rh   ri   rs   rf   rï   N©rq   r&  r'  r(  r)  r5  r*  r%  r(   r&   rõ   rõ   M  s   † ñð, Ø÷-r(   rõ   c                   ó>   • \ rS rSrSr  SS\S-  S\S-  4S jjrSrg)	rŠ   iŒ  a'  
Single squad example features to be fed to a model. Those features are model-specific and can be crafted from
[`~data.processors.squad.SquadExample`] using the
:method:*~transformers.data.processors.squad.squad_convert_examples_to_features* method.

Args:
    input_ids: Indices of input sequence tokens in the vocabulary.
    attention_mask: Mask to avoid performing attention on padding token indices.
    token_type_ids: Segment token indices to indicate first and second portions of the inputs.
    cls_index: the index of the CLS token.
    p_mask: Mask identifying tokens that can be answers vs. tokens that cannot.
        Mask with 1 for tokens than cannot be in the answer and 0 for token that can be in an answer
    example_index: the index of the example
    unique_id: The unique Feature identifier
    paragraph_len: The length of the context
    token_is_max_context:
        List of booleans identifying which tokens have their maximum context in this feature object. If a token
        does not have their maximum context in this feature object, it means that another feature object has more
        information related to that token and should be prioritized over this feature for that token.
    tokens: list of tokens corresponding to the input ids
    token_to_orig_map: mapping between the tokens and the original text, needed in order to identify the answer.
    start_position: start of the answer token index
    end_position: end of the answer token index
    encoding: optionally store the BatchEncoding with the fast-tokenizer alignment methods.
Nri   r  c                 óÆ   • Xl         X l        X0l        X@l        XPl        X`l        Xpl        X€l        X�l        X l	        X°l
        XÀl        XÐl        Xàl        Xðl        UU l        g rº   )rY   rc   ra   r®   r¯   rd   re   rZ   r^   r[   r\   rf   rg   rh   ri   r  )rö   rY   rc   ra   r®   r¯   rd   re   rZ   r^   r[   r\   rf   rg   rh   ri   r  s                    r&   r5  ÚSquadFeatures.__init__§  s_   € ð& #ŒØ,ÔØ,ÔØ"ŒØŒà*ÔØ"ŒØ*ÔØ$8Ô!ØŒØ!2Ôà,ÔØ(ÔØ*ÔØŒà ˆ�r(   )rc   r®   r  rg   rd   rY   rh   r¯   rZ   ri   rf   r^   r\   ra   r[   re   )NN)	rq   r&  r'  r(  r)  Ústrr
   r5  r*  r%  r(   r&   rŠ   rŠ   Œ  s7   † ñðT "Ø)-ñ#%!ð  �d‘
ð!%!ð"   $Ñ&÷#%!ð %!r(   rŠ   c                   ó"   • \ rS rSrSrSS jrSrg)ÚSquadResultiÏ  a2  
Constructs a SquadResult which can be used to evaluate a model's output on the SQuAD dataset.

Args:
    unique_id: The unique identifier corresponding to that example.
    start_logits: The logits corresponding to the start of the answer
    end_logits: The logits corresponding to the end of the answer
Nc                 ó\   • X l         X0l        Xl        U(       a  X@l        XPl        X`l        g g rº   )Ústart_logitsÚ
end_logitsre   Ústart_top_indexÚend_top_indexÚ
cls_logits)rö   re   r?  r@  rA  rB  rC  s          r&   r5  ÚSquadResult.__init__Ù  s+   € Ø(ÔØ$ŒØ"ŒæØ#2Ô Ø!.ÔØ(�Oð r(   )rC  r@  rB  r?  rA  re   )NNNr7  r%  r(   r&   r=  r=  Ï  s   † ñ÷)r(   r=  )rP   Fr   T)+r  r
  Ú	functoolsr   Úmultiprocessingr   r   Úmultiprocessing.poolr   rò   rƒ   r   Ú$models.bert.tokenization_bert_legacyr	   Útokenization_utils_baser
   r   r   Úutilsr   r   r   r   rw   rÉ   Útorch.utils.datar   Ú
get_loggerrq   rl   r'   r<   r>   rE   r·   r»   rä   ræ   r,  r0  rõ   rŠ   r=  r%  r(   r&   Ú<module>rM     sà   ðó Û 	Ý ß +Ý +ã Ý å Gß aÑ aß HÑ HÝ  ò #LÐ ñ ×ÑÛÝ.ð 
×	Ò	˜HÓ	%€ò
$ò-ò(-ò,òJðZ&ÐBYô &ð "ØØØôuôpO�]ô Oôd�~ô ô
�~ô ÷
<ñ <÷~@!ñ @!÷F)ò )r(   