ó
    6lju8  ã                  ó¤  • S r SSKJr  SSKrSSKrSSKrSSKrSSKrSSKJ	r	  SSK
Jr  SSKJrJrJrJr  SSKJr  SSKJrJr  SS	KJr  SS
KJr  SSKJrJrJr  SSKJ r J!r!  SSK"J#r#J$r$  \" S5      r%Sr&Sr'Sr( " S S\5      r)S"S jr*S#S jr+S$S jr,S%S jr-S&S jr.S'S jr/S(S jr0S)S jr1S*S jr2S+S jr3S,S jr4S-S jr5S.S  jr6\7S!:X  a  \6" 5         gg)/u›  Measure four ways of returning an answer over one set of questions.

Every arm answers from the SAME retrieved context: retrieval runs once per
question and its chunks are handed to all four, so a difference in the results
is a difference in the return shape and not in what was retrieved.

The arms:

  baseline  the playground's current agent â€” prose plus cited ids
  fixed     one envelope, payload chosen from the closed union in `shapes`
  planned   a planner declares fields per question; they are compiled and applied
  declared  the caller states the shape; nothing is inferred

Scoring is deterministic. Gold values were copied out of printed table cells,
so a returned figure is compared to the gold string directly and, separately,
as a number â€” no judge is asked whether two figures agree. The baseline has no
value field to compare, so it is scored on whether the gold figure appears
anywhere in its prose: that measures whether the information was found, which
is the point the comparison has to be fair about.

Usage:
    python -m quber.playground.benchmark.run_shapes --gold <path>
    python -m quber.playground.benchmark.run_shapes --gold <path> --limit 4
é    )ÚannotationsN)ÚDecimal)ÚPath)ÚAnyÚDictÚListÚOptional)Úlogger)Ú	BaseModelÚField)Údb)Úanswer)ÚdeclaredÚfixedÚplanned)Ú	as_numberÚnorm_printed)ÚRetrievedChunkÚretrievez#output/benchmark/answer_shapes.jsoné   é
   )Úbaseliner   r   r   c                  óF   • \ rS rSr% Sr\" SS9rS\S'   \" \S9r	S\S	'   S
r
g)ÚProseAnsweré5   z9The shape a caller declares when it wants an explanation.zThe explanation, in prose.©ÚdescriptionÚstrÚtext)Údefault_factoryz	List[str]Ú	cited_ids© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r   Ú__annotations__Úlistr!   Ú__static_attributes__r"   ó    ÚC/home/mande/repo/quber/src/quber/playground/benchmark/run_shapes.pyr   r   5   s$   ‡ ÙCáÐ">Ñ?€Dˆ#Ó?Ù °Ñ6€IˆyÖ6r+   r   c                ó¶   • U c  SSS.$ [        U 5      [        U5      :H  n[        U 5      [        U5      pCU[        USL=(       a    USL=(       a    X4:H  5      S.$ )zJExact-as-printed and numeric agreement between a returned and gold figure.NF)ÚexactÚnumeric)r   r   Úbool)ÚgotÚgoldr.   ÚaÚbs        r,   Úvalues_agreer5   D   s[   € à
�{Ø¨5Ñ1Ð1Ü˜Ó¤¨dÓ!3Ñ3€EÜ�S‹>œ9 T›?€qØ¤t¨A°T¨M×,V¸aÀt¸m×,VÐPQÑPVÓ'WÑXÐXr+   c                óŠ   • U (       d  g[        U 5      n[        U5      S:„  =(       d    [        UR                  5       5      S:¬  $ )u  True when a value field holds a sentence rather than a figure.

A printed figure is short and at most a few tokens ('$1.2 million',
'4.6x'). Four or more tokens, or a long string, means the value came back
wrapped in words â€” the failure this whole exercise is about.
Fé(   r   )r   ÚlenÚsplit)ÚsÚts     r,   Úlooks_like_proser<   M   s5   € ö ØÜ�Q‹€AÜˆq‹6�B‰;×-œ#˜aŸg™g›i›.¨AÑ-Ð-r+   c                óö   • U (       d  g[         R                  " SSU5      [         R                  " SSU 5      ;   a  g[        U5      nUc  g[         R                  " SU 5       H  n[        U5      U:X  d  M    g   g)u÷  Whether the gold figure appears in a prose answer, as printed or as a number.

Whitespace is ignored on the literal comparison: a cell prints "$ (10,550)"
where prose writes "$(10,550)", and that is the same figure.

Scale words are deliberately not part of a token. Both sides drop the scale
when they parse â€” "$241.4M" and "$241.4 million" each read as 241.4 â€” so
absorbing a trailing "thousand" into the token achieves nothing and breaks
any figure whose closing parenthesis then sits mid-token.
Fz\s+Ú Tz)\(?-?\$?\(?[\d][\d,]*(?:\.\d+)?\)?%?[xX]?)ÚreÚsubr   Úfindall)Úproser2   ÚgÚtoks       r,   Úgold_in_proserE   Z   sm   € ö ØÜ	‡v‚vˆf�b˜$Ó¤2§6¢6¨&°"°eÓ#<Ó<ØÜ�$‹€AØ�yØô
 �zŠzÐFÈÖNˆÜ�S‹>˜QÕÙñ Oð r+   c           	     ó°  • [         R                  " 5        nUR                  SU SU S345      R                  5       nSSS5        W(       d  g[        R
                  " S[        R                  " U5       S3US   [        R                  5      nU(       a0  [        [        R                  " SS	UR                  S
5      5      5      $ S$ ! , (       d  f       N�= f)zHThe text printed in one tagged table cell of one document, if it exists.zÅSELECT c.content FROM ade_playground.chunks c
               JOIN ade_playground.documents d ON d.id = c.document_id
               WHERE d.doc_key = %s AND c.content LIKE %s
               LIMIT 1z%id="z"%Nz<td id="z"[^>]*>(.*?)</td>r   z<[^>]+>r>   é   )r   ÚconnectÚexecuteÚfetchoner?   ÚsearchÚescapeÚSr   r@   Úgroup)Údoc_keyÚcell_idÚconnÚrowÚms        r,   Ú	cell_textrT   y   s©   € ä	�ŠŒ˜Ø�l‰lðð ˜˜g˜Y bÐ)Ð*ó
÷ ‰(‹*ð 	÷ 
ö ØÜ
�	Š	�XœbŸiši¨Ó0Ð1Ð1BÐCÀSÈÁVÌRÏTÉTÓR€AÞ?@Œ<œŸš˜z¨2¨q¯w©w°q«zÓ:Ó;ÐJÀdÐJ÷ 
�ús   –'CÃ
Cc                ó.   ^• [        U4S jU  5       5      $ )Nc              3  óJ   >#   • U  H  nS T S3UR                   ;   v •  M     g7f)zid="Ú"N)Úcontent)Ú.0ÚcrP   s     €r,   Ú	<genexpr>Ú retrieval_hit.<locals>.<genexpr>Š   s#   øé € Ð>²v°!��g�Y˜aÐ  A§I¡IÖ-²vùs   ƒ #)Úany)ÚchunksrP   s    `r,   Úretrieval_hitr_   ‰   s   ø€ ÜÔ>±vÓ>Ó>Ð>r+   c                ód  • U c  g[        U [        5      (       a  U R                  5       O
[        U 5      nUR	                  5        Hh  u  p#US;   a  M  [        U[
        5      (       a  UR                  5       (       a  Us  $ [        U[        [        [        45      (       d  M]  [        U5      s  $    g)zéThe single figure a compiled or declared model returned, if it has one.

A planner-compiled model names its field whatever suited the question, so
the figure is found by taking the model's first string field that is not
bookkeeping.
N)r!   Ú	not_found)
Ú
isinstancer   Ú
model_dumpÚdictÚitemsr   ÚstripÚintÚfloatr   )ÚobjÚdataÚkeyÚvals       r,   Úscalar_fieldrm   �   s†   € ð �{ØÜ)¨#¬y×9Ñ9ˆ3�>‰>Ô¼tÀC»y€DØ—J‘J–L‰ˆØÐ,Ó,ÙÜ�cœ3×Ñ C§I¡I§K¡KØŠJÜ�cœC¤¬Ð0×1Ó1Ü�s“8ŠOñ !ð r+   c           	   ƒ  óÎ  #   • [         R                  " 5       nSU 0nUS   S:H  n U S:X  a?  [        US   U5      I Sh  v•N nUR                  SSUR                  UR
                  S9  GO÷U S	:X  aÿ  [        R                  " US   U5      I Sh  v•N nUR                  nUR                  UR                  UR
                  S
9  UR                  S:X  aR  UR                  UR                  UR                  UR                  UR                  UR                  UR                  S9  GO=UR                  S:X  a  UR                  SUR                  S9  GOUR                  SUR!                  SS9S9  GOòU S:X  Ga  ["        R$                  " US   5      I Sh  v•N nUR!                  5       US'   UR&                  (       a†  UR(                  (       au  ["        R*                  " U5      n	[,        R                  " US   X)5      I Sh  v•N nUR!                  5       n
UR                  S[/        U5      U
R1                  S/ 5      U
S9  GO [,        R                  " US   U[2        5      I Sh  v•N nUR                  SSUR                  UR
                  S9  OÒU S:X  aÌ  U(       ax  [,        R                  " US   U[,        R4                  5      I Sh  v•N nUR                  SUR                  UR                  UR                  UR
                  UR6                  S9  OM[,        R                  " US   U[2        5      I Sh  v•N nUR                  SSUR                  UR
                  S9  [?        [         R                  " 5       U-
  S5      US'   [A        XA5      $  GNT GN GN GN› GN: N× Nk! [8         a4  nUR                  [;        U5      R<                   SU 3SS/ S9   SnAN|SnAff = f7f)zNOne arm's attempt at one question, with what it returned and how long it took.ÚarmÚkindÚvaluer   ÚquestionNrB   )Úshaperq   rB   r!   r   )rs   r!   Úscalar)rq   ÚnumberÚunitÚperiodÚlabelÚ	source_id)rq   rB   Újson)Úmode)rq   Úpayloadr   Úplanr!   )rs   rq   r!   Úfieldsr   )rs   rq   rv   rw   r!   ra   z: )Úerrorrs   rq   r!   é   Úseconds)!ÚtimeÚperf_counterÚbaseline_answerÚupdater   r!   r   r|   rp   rq   ru   rv   rw   rx   ry   r   rc   r   Úplan_forÚwants_valuer~   Úcompile_modelr   rm   Úgetr   ÚDeclaredScalarra   Ú	ExceptionÚtyper#   ÚroundÚscore)ro   Úqr^   ÚstartedÚoutÚis_valueÚresr|   r}   Úmodelrj   Úexcs               r,   Úrun_armr–   ¤   s@  é € ä×ÒÓ!€GØ  #˜,€CØ�‰y˜GÑ#€HðB_Ø�*ÓÜ'¨¨*©°vÓ>×>ˆCØ�J‰JØØØ—j‘jØŸ-™-ð	 ó ð �G‹^ÜŸš Q z¡]°FÓ;×;ˆCØ—k‘kˆGØ�J‰J˜WŸ\™\°S·]±]ˆJÑCØ�|‰|˜xÓ'ð
 —
‘
Ø!Ÿ-™-Ø"Ÿ>™>Ø Ÿ™Ø"Ÿ>™>Ø!Ÿ-™-Ø%×/Ñ/ð ó ð —‘ Ó(Ø—
‘
 ¨W¯\©\�
Ó:à—
‘
 ¨w×/AÑ/AÀvÐ/AÐ/N�
ÓOà�IÔÜ ×)Ò)¨!¨J©-Ó8×8ˆDØŸ/™/Ó+ˆC�‰KØ×× D§K§KÜ×-Ò-¨dÓ3�Ü$ŸOšO¨A¨j©M¸6ÓI×I�Ø—~‘~Ó'�Ø—
‘
Ø!Ü& sÓ+Ø"Ÿh™h {°BÓ7Øð	 ó ô %ŸOšO¨A¨j©M¸6Ä;ÓO×O�Ø—
‘
 °¸C¿H¹HÐPS×P]ÑP]�
Ò^à�JÓæÜ$ŸOšO¨A¨j©M¸6Ä8×CZÑCZÓ[×[�Ø—
‘
Ø!ØŸ)™)ØŸ™ØŸ:™:Ø!Ÿm™mØ!Ÿm™mð ò ô %ŸOšO¨A¨j©M¸6Ä;ÓO×O�Ø—
‘
 °¸C¿H¹HÐPS×P]ÑP]�
Ñ^ô
 œ4×,Ò,Ó.°Ñ8¸!Ó<€Cˆ	�NÜ�‹=ÐòG ?ò <ò, 9ò Jò Pñ \ñ Pøô ó _Ø�
‰
œD ›I×.Ñ.Ð/¨r°#°Ð7¸tÈ4Ð[]ˆ
Ö^ûð_üs  ‚"O%¥N$ ½N¾*N$ Á(O%Á*#N$ ÂNÂBN$ Ä"O%Ä$*N$ ÅO%ÅN$ Å-O%Å/#N$ ÆNÆA,N$ Ç?NÈ ?N$ È?O%É"N$ É#NÉ$*N$ ÊO%Ê9N$ ËN Ë	A
N$ ÌO%Ì"N$ Ì6N"Ì7*N$ Í!0O%ÎN$ ÎN$ ÎN$ ÎN$ ÎN$ Î N$ Î"N$ Î$
O"Î.*OÏO%ÏO"Ï"O%c                óä  • US   S:H  nU R                  S5      nU(       a  US;   OUS;   U S'   [        U R                  S5      5      U S'   U(       až  US	   nU R                  S5      bE  U R                  [        U S   U5      5        [	        U S   5      U S
'   U S   =(       d    U S   U S'   U $ U R                  S5      =(       d    SnU R                  SS[        U5      S9  [        XT5      U S'   U $ )z.Add the measurements this comparison turns on.rp   rq   rs   )rt   rq   )rB   ÚunanswerableÚshape_correctr!   Ú	cites_anyÚ
gold_valueÚ
prose_leakr.   r/   Ú
found_goldrB   r>   F)r.   r/   rœ   )r‰   r0   r…   r5   r<   rE   )r‘   r�   r’   rs   r2   rB   s         r,   rŽ   rŽ   ñ   sò   € à�‰y˜GÑ#€HØ�G‰G�GÓ€EÞ;C˜5Ð$7Ò7ÈÐRkÑIk€CˆÑÜ˜CŸG™G KÓ0Ó1€CˆÑæØ�‰ˆØ�7‰7�7ÓÑ'Ø�J‰J”| C¨¡L°$Ó7Ô8Ü 0°°W±Ó >ˆC�ÑØ # G¡× >°°I±ˆC�Ñð €Jð —G‘G˜GÓ$×*¨ˆEØ�J‰J˜U¨E¼dÀ5»kˆJÑJÜ -¨eÓ :ˆC�ÑØ€Jr+   c           	   ƒ  ó8  #   • U IS h  v•N   [        U S   U S   [        S9I S h  v•N nS S S 5      IS h  v•N   0 U E[        W Vs1 s H  o3R                  S-   iM     sn5      U S   S:X  a  [	        X S   5      OS S.En/ n[
         HA  nU IS h  v•N   UR                  [        X`U5      I S h  v•N 5        S S S 5      IS h  v•N   MC     XTS	'   U$  NÊ N¯ N¡! , IS h  v•N  (       d  f       N¶= fs  snf  Nf NI N7! , IS h  v•N  (       d  f       MŽ  = f7f)
NÚslugrr   )ÚkrG   rp   rq   Úgold_cell_id)Úretrieved_pagesr_   Úarms)r   ÚKÚsortedÚpager_   ÚARMSÚappendr–   )r�   Úsemr^   rZ   rR   Úresultsro   s          r,   Úrun_questionr«   	  så   é € ßŠsÜ  &¡	¨1¨Z©=¼AÑ>×>ˆ÷ �sðØ
ðä!±vÓ">²v°!§6¡6¨A¤:±vÑ">Ó?ØEFÀvÁYÐRYÓEYœ v°Ñ/@ÔAÐ_cò€Cð
 €Gßˆß’3Ø�N‰N¤¨°Ó!8×8Ô9÷ —3‘3ñ ð ˆ�KØ€Jñ Ù>÷ �s�sˆsüò #?ñ
 Ù8÷ —3—3‘3üs±   ‚D‰CŠD�C¦C§C«D¶C·DÁC4
Á3DÂC9ÂDÂC?Â1C;Â2C?Â:DÃC=ÃDÃCÃDÃC1Ã C#Ã!C1Ã-DÃ;C?Ã=DÃ?D	ÄDÄD	ÄDc                óV  • U  Vs/ s H  oS   S:X  d  M  UPM     nnU Vs/ s H  oS   (       d  M  UPM     nnU  Vs/ s H  oS   S:X  d  M  UPM     nn[        U5      [        U5      [        U5      0 S.nSS jn[         GH  nU Vs/ s H
  o" X5      PM     nnU Vs/ s H
  o" X5      PM     n	n[        U5      =(       d    Sn
[        S U 5       5      [        S	 U 5       5      [        S
 U 5       5      [        S U 5       5      [        S U 5       5      [        S U 5       5      [        S U	 5       5      [        S U 5       5      [        S X‰-    5       5      [        [        S U 5       5      U
-  S5      S.
US   U'   GM     U$ s  snf s  snf s  snf s  snf s  snf )z?Per-arm totals over the value questions and the prose controls.rp   rq   r_   rB   )Úvalue_questionsÚ"value_questions_with_retrieval_hitÚprose_controlsr£   c                ó4   ^• [        U4S jU S    5       5      $ )Nc              3  ó<   >#   • U  H  oS    T:X  d  M  Uv •  M     g7f)ro   Nr"   )rY   r3   ro   s     €r,   r[   Ú,summarize.<locals>.arm_of.<locals>.<genexpr>&  s   øé € Ð>š{˜!°©h¸#©o—A‘Aš{ùs   ƒ“	r£   )Únext)rR   ro   s    `r,   Úarm_ofÚsummarize.<locals>.arm_of%  s   ø€ ÜÔ>˜s 6š{Ó>Ó>Ð>r+   rG   c              3  óD   #   • U  H  oR                  S S5      v •  M     g7f)r.   FN©r‰   ©rY   r3   s     r,   r[   Úsummarize.<locals>.<genexpr>-  s   é € Ð)LÊÀA¯%©%°¸×*?Ð*?Êùó   ‚ c              3  óD   #   • U  H  oR                  S S5      v •  M     g7f)r/   FNr·   r¸   s     r,   r[   r¹   .  s   é € Ð&KÊÀ1§u¡u¨Y¸×'>Ð'>Êùrº   c              3  óD   #   • U  H  oR                  S S5      v •  M     g7f)r�   FNr·   r¸   s     r,   r[   r¹   /  s   é € Ð&NÊ2Àa§u¡u¨\¸5×'AÐ'AÊ2ùrº   c              3  óF   #   • U  H  oR                  S 5      SLv •  M     g7f)rq   Nr·   r¸   s     r,   r[   r¹   0  s   é € Ð)QÊbÈ¯%©%°«.ÀÕ*DÊbùs   ‚!c              3  óD   #   • U  H  oR                  S S5      v •  M     g7f)rœ   FNr·   r¸   s     r,   r[   r¹   1  s   é € ÐEÂ"¸QŸe™e L°%×8Ð8Â"ùrº   c              3  ó*   #   • U  H	  oS    v •  M     g7f©r™   Nr"   r¸   s     r,   r[   r¹   2  s   é € Ð*JÂrÀ!¨_Ö+=Ârùó   ‚c              3  ó*   #   • U  H	  oS    v •  M     g7frÀ   r"   r¸   s     r,   r[   r¹   3  s   é € Ð)IÂbÀ¨OÖ*<ÂbùrÁ   c              3  ó*   #   • U  H	  oS    v •  M     g7f)rš   Nr"   r¸   s     r,   r[   r¹   4  s   é € Ð8²R°˜{ž^²RùrÁ   c              3  ó,   #   • U  H
  nS U;   v •  M     g7f)r   Nr"   r¸   s     r,   r[   r¹   5  s   é € Ð8²¨1˜' Qž,²ùs   ‚c              3  ó*   #   • U  H	  oS    v •  M     g7f)r�   Nr"   r¸   s     r,   r[   r¹   6  s   é € Ð%?ºB°q¨	¦lºBùrÁ   r€   )
Úvalue_exact_as_printedÚvalue_numeric_matchÚfound_gold_anywhereÚreturned_a_value_fieldrœ   Úshape_correct_on_valuesÚshape_correct_on_proserš   ÚerrorsÚmean_secondsr£   )rR   úDict[str, Any]ro   r   ÚreturnrÎ   )r8   r§   Úsumr�   )ÚrowsÚrÚ
value_rowsÚhit_rowsÚ
prose_rowsr‘   r´   ro   ÚvsÚpsÚns              r,   Ú	summarizerÙ     sˆ  € á!Ó:šT˜ v¡Y°'Ñ%9—!™T€JÐ:Ù%Ó<š:�a¨?Õ);—™:€HÐ<Ù!Ó:šT˜ v¡Y°'Ñ%9—!™T€JÐ:ä˜z›?Ü.1°(«mÜ˜j›/Øñ	€Cô?÷ ˆtˆÙ&.Ó/¢h ˆf�QŽn¡hˆÐ/Ù&0Ó1¢j ˆf�QŽn¡jˆÐ1Ü�‹G�L�qˆä&)Ñ)LÉÓ)LÓ&LÜ#&Ñ&KÉÓ&KÓ#KÜ#&Ñ&NÉ2Ó&NÓ#NÜ&)Ñ)QÉbÓ)QÓ&QÜÑEÁ"ÓEÓEÜ'*Ñ*JÁrÓ*JÓ'JÜ&)Ñ)IÁbÓ)IÓ&IÜÑ8±RÓ8Ó8ÜÑ8°²Ó8Ó8Ü!¤#Ñ%?¹BÓ%?Ó"?À!Ñ"CÀQÓGñ
ˆˆF‰�CÔñ	 ð  €Jùò; ;ùÚ<ùÚ:ùò 0ùÚ1s+   …F•F¡F²F¾FÁFÂF!Â"F&c              ƒ  ó|  ^#   • [         R                  " U R                  5       5      nU(       a  US U OUn[        R                  " S[        U5      [        [        5      S9  [        R                  " [        5      m[        R                  " U4S jU 5       6 I S h  v•N n[        [        U5      5      n[        R                  R                  SSS9  [        R!                  [         R"                  " XTS.S[$        S95        [        R&                  " S	[        S
9  [)        [         R"                  " USS95        g  N�7f)Nz{n} questions x {a} arms)rØ   r3   c              3  ó<   >#   • U  H  n[        UT5      v •  M     g 7f)N)r«   )rY   r�   r©   s     €r,   r[   Úrun.<locals>.<genexpr>@  s   øé € Ð!JÂ	¸1¤,¨q°#×"6Ð"6Â	ùs   ƒT)ÚparentsÚexist_ok)ÚsummaryrÑ   r€   ©ÚindentÚdefaultzwritten to {p}©Úp©rá   )rz   ÚloadsÚ	read_textr
   Úinfor8   r§   ÚasyncioÚ	SemaphoreÚCONCURRENCYÚgatherrÙ   r)   ÚOUTPUTÚparentÚmkdirÚ
write_textÚdumpsr   ÚsuccessÚprint)Ú	gold_pathÚlimitr2   Ú	questionsrÑ   rß   r©   s         @r,   Úrunr÷   ;  sØ   øé € Ü�:Š:�i×)Ñ)Ó+Ó,€DÞ %��V�e‘¨4€IÜ
‡K‚KÐ*¬c°)«nÄÄDÃ	ÒJÜ
×
Ò
œKÓ
(€CÜ—’Ô!JÁ	Ó!JÐK×K€DÜœ˜T›
Ó#€Gä
‡M�M×Ñ ¨tÐÑ4Ü
×Ñ”d—j’j¨WÑ!CÈAÔWZÑ[Ô\Ü
‡N‚NÐ#¤vÒ.Ü	Œ$�*Š*�W QÑ
'Õ(ñ Lùs   ƒBD<ÂD:ÂBD<c                ó’  • [         R                  " U R                  5       5      nUS    H%  nUS    Vs/ s H  n[        X25      PM     snUS'   M'     [	        US   5      US'   U R                  [         R                  " US[        S95        [        R                  " SU S9  [        [         R                  " US   SS95        g	s  snf )
u   Recompute the measurements over a finished run's stored answers.

Every arm's raw output is kept in the results file, so a correction to the
scoring is applied by re-reading it â€” no model is called again, and the
answers being scored are byte-identical to the ones originally returned.
rÑ   r£   rß   r€   rà   zrescored {p}rã   rå   N)rz   ræ   rç   rŽ   rÙ   rð   rñ   r   r
   rò   ró   )Úpathrj   rR   ro   s       r,   Úrescorerú   I  sœ   € ô �:Š:�d—n‘nÓ&Ó'€DØ�FŒ|ˆØ25°f²+Ó>²+¨3”u˜S–±+Ñ>ˆˆF‹ñ ä  V¡Ó-€Dˆ�OØ‡O�O”D—J’J˜t¨A´sÑ;Ô<Ü
‡N‚N�> TÒ*Ü	Œ$�*Š*�T˜)‘_¨QÑ
/Õ0ùò	 ?s   ´Cc                 óÌ  • [         R                  " [        S9n U R                  S[        SS9  U R                  S[
        S S9  U R                  S[        SS9  U R                  5       nUR                  (       a  [        UR                  5        g UR                  (       d  U R                  S	5        [        R                  " [        UR                  UR                  5      5        g )
Nr   z--goldzgold question set JSON)rŒ   Úhelpz--limit)rŒ   râ   z	--rescorez$recompute scores over a finished runz,--gold is required unless --rescore is given)ÚargparseÚArgumentParserr'   Úadd_argumentr   rg   Ú
parse_argsrú   r2   r   ré   r÷   rõ   )rä   Úargss     r,   Úmainr  Y  sš   € Ü×Ò¬GÑ4€AØ‡N�N�8¤$Ð-E€NÑFØ‡N�N�9¤3°€NÑ5Ø‡N�N�;¤TÐ0V€NÑWØ�<‰<‹>€DØ‡|‡|Ü�—‘ÔØØ�9�9Ø	�‰Ð>Ô?Ü‡K‚K”�D—I‘I˜tŸz™zÓ*Õ+r+   Ú__main__)r1   úOptional[str]r2   r   rÏ   zDict[str, bool])r:   r  rÏ   r0   )rB   r   r2   r   rÏ   r0   )rO   r   rP   r   rÏ   r  )r^   úList[RetrievedChunk]rP   r   rÏ   r0   )ri   r   rÏ   r  )ro   r   r�   rÎ   r^   r  rÏ   rÎ   )r‘   rÎ   r�   rÎ   rÏ   rÎ   )r�   rÎ   r©   zasyncio.SemaphorerÏ   rÎ   )rÑ   zList[Dict[str, Any]]rÏ   rÎ   )rô   r   rõ   zOptional[int]rÏ   ÚNone)rù   r   rÏ   r  )rÏ   r  )8r'   Ú
__future__r   rý   ré   rz   r?   r‚   Údecimalr   Úpathlibr   Útypingr   r   r   r	   Úlogurur
   Úpydanticr   r   Úquber.playgroundr   Úquber.playground.agentr   r„   Úquber.playground.answersr   r   r   Ú quber.playground.answers.figuresr   r   Úquber.playground.retrievalr   r   rí   rë   r¤   r§   r   r5   r<   rE   rT   r_   rm   r–   rŽ   r«   rÙ   r÷   rú   r  r#   r"   r+   r,   Ú<module>r     sÂ   ðñõ2 #ã Û Û Û 	Û Ý Ý ß ,Ó ,å ß %å Ý <ß =Ñ =ß Dß ?á	Ð3Ó	4€Ø€Ø€à3€ô7�)ô 7ôYô
.ôô>Kô ?ôô(JôZô0ô ôD)ô1ô ,ð ˆzÓÙ…Fð r+   