ó
    6lj°  ã                  óâ   • S r SSKJr  SSKrSSKrSSKrSSKJr  SSKJ	r	J
r
JrJrJrJr  SSKJr  SSKJr  SS jrSS	 jrSS
 jrSS jrSS jr\4     SS jjrSS jr\S:X  a  \" 5         gg)u…  Check every authored gold value against the cell it claims to come from.

The questions were authored by agents reading printed tables, so each one
names a table cell and quotes what that cell prints. This reads the cell out
of the database and compares. A question whose gold value does not match its
cell is dropped rather than corrected â€” a benchmark answer that had to be
repaired is not a benchmark answer.

Input is the authoring workflow's journal; output is the gold set the
comparison runs against.

Usage:
    python -m quber.playground.benchmark.verify_gold \
        --journal <path>/journal.jsonl --out output/benchmark/gold.json
é    )ÚannotationsN)ÚPath)ÚAnyÚCallableÚDictÚListÚOptionalÚTuple)Úlogger)Údbc                ó`   • [         R                  " SSU =(       d    SR                  5       5      $ )Nz\s+Ú Ú )ÚreÚsubÚstrip)Úss    ÚD/home/mande/repo/quber/src/quber/playground/benchmark/verify_gold.pyÚnormr      s    € Ü�6Š6�&˜# § RŸ™Ó0Ó1Ð1ó    c                ó"  • / nU R                  5       R                  5        Hj  n[        R                  " U5      nUR	                  S5      S:X  d  M0  [        UR	                  S5      [        5      (       d  MV  UR                  US   5        Ml     U$ )NÚtypeÚresult)Ú	read_textÚ
splitlinesÚjsonÚloadsÚgetÚ
isinstanceÚdictÚappend)ÚjournalÚdocsÚlineÚrows       r   Úload_documentsr&   "   sm   € Ø€DØ×!Ñ!Ó#×.Ñ.Ö0ˆÜ�jŠj˜ÓˆØ�7‰7�6‹?˜hÕ&¬:°c·g±g¸hÓ6GÌ×+NÓ+NØ�K‰K˜˜H™Ö&ñ 1ð €Kr   c                óØ   • [         R                  " SU =(       d    S[         R                  5       VVs/ s H(  u  pU[        [         R                  " SSU5      5      4PM*     snn$ s  snnf )zFThe id-tagged cells of one chunk's markup, as (cell id, printed text).z <td id="([^"]+)"[^>]*>(.*?)</td>r   z<[^>]+>)r   ÚfindallÚSr   r   )ÚcontentÚcidÚinners      r   Útagged_cellsr-   +   s\   € ô Ÿ*š*Ð%HÈ'Ï-ÐUWÔY[×Y]ÑY]Ô^ôâ^‰JˆCð 
Œd”2—6’6˜* b¨%Ó0Ó1Ó2Ù^òð ùó s   ³/A&c                ó6  • [         R                  " 5        nUR                  SU 45      R                  5       nSSS5        0 nW H@  u  pE[	        U5       H,  u  pgUR                  XF4U5        UR                  SU4U5        M.     MB     U$ ! , (       d  f       NX= f)zMEvery tagged cell of every chunk in one document, keyed by chunk and cell id.z¤SELECT c.chunk_id, c.content FROM ade_playground.chunks c
               JOIN ade_playground.documents d ON d.id = c.document_id
               WHERE d.doc_key = %sNr   )r   ÚconnectÚexecuteÚfetchallr-   Ú
setdefault)Údoc_keyÚconnÚrowsÚindexÚchunk_idr*   r+   Útexts           r   Ú
cell_indexr9   3   s’   € ä	�ŠŒ˜Ø�|‰|ð'ð ˆJó	
÷
 ‰(‹*ð 	÷ 
ð )+€EÛ!ÑˆÜ% gÖ.‰IˆCØ×Ñ˜h˜_¨dÔ3Ø×Ñ˜b #˜Y¨Ö-ó /ñ "ð €L÷ 
�ús   –"B
Â

Bc           
     óª   • [        U5      n[        U R                  5        VVVs1 s H  u  u  p4oS(       a  M  XR:X  d  M  UiM     snnn5      $ s  snnnf )a¢  How many distinct cells in the document print this exact figure.

Counted over cell ids, not over chunks: a table's cells are repeated in
each of its line-item chunks under the same ids, so counting chunk
entries would report every figure as duplicated. Reported so a reviewer
can see which questions rest on a figure that appears more than once,
where a right-looking answer read off the wrong row would still score.
)r   ÚlenÚitems)r6   ÚprintedÚtargetÚchunkr+   r8   s         r   Úoccurrencesr@   D   s?   € ô �'‹]€FÜ¨e¯k©k¬mÕ\ªmÑ*™˜ dÃ5“ÈTÉ^—©mÓ\Ó]Ð]ùÔ\s   ¥A¸A¿Ac                ó"  • / n/ nU  GH‚  nUS   nU" U5      nUR                  S/ 5       GH  nUS   nUR                  SS5      n	UR                  X˜45      =(       d    UR                  SU45      n
[        US   5      nU
c   UR                  0 UEUSU SU 3S	.E5        Mx  X«:w  a%  UR                  0 UEUSU S
U
< SU< 3S	.E5        M¢  UR                  USUS   U
UU	UR                  S5      UR                  S5      UR                  S5      UR                  S5      UR                  S5      [        Xj5      S.5        GM     UR                  S/ 5       H-  nUR                  USUS   S S UR                  S5      S.5        M/     GM…     X#4$ )NÚslugÚvalue_questionsÚgold_cell_idr7   r   Ú
gold_valuezcell z not found in )rB   Úwhyz prints z, gold says ÚvalueÚquestionÚpageÚunitÚperiodÚstyleÚuniqueness_note)rB   ÚkindrH   rE   rD   r7   rI   rJ   rK   rL   rM   Úoccurrences_in_documentÚprose_questionsÚproseÚ	why_prose)rB   rN   rH   rE   rD   rR   )r   r   r!   r@   )r#   Ú	index_forÚkeptÚdroppedÚdocrB   r6   Úqr+   r?   r=   Úgolds               r   ÚverifyrY   Q   s°  € ð "$€DØ$&€GÜˆØ�6‰{ˆÙ˜$“ˆØ—‘Ð*¨B×/ˆAØ�NÑ#ˆCØ—E‘E˜* bÓ)ˆEØ%*§Y¡Y°¨|Ó%<×%TÀÇ	Á	È2ÈsÈ)Ó@TˆGÜ˜˜,™Ó(ˆDØ‰Ø—‘Ð\ !Ð\¨T¸EÀ#ÀÀnÐUYÐTZÐ:[Ò\Ô]ÙØ‹Ø—‘Ød�qÐd $°°s°e¸8ÀGÁ;ÈlÐ[_ÑZbÐ/cÒdôñ Ø�K‰Kà Ø#Ø ! *¡Ø")Ø$'Ø %ØŸE™E &›MØŸE™E &›MØŸe™e H›oØŸU™U 7›^Ø'(§u¡uÐ->Ó'?Ü/:¸5Ó/Jñ÷ñ 0ð: —‘Ð*¨BÖ/ˆAØ�K‰Kà Ø#Ø ! *¡Ø"&Ø$(Ø!"§¡ {Ó!3ñö	ô 0ñA ðV ˆ=Ðr   c            	     óÎ  • [         R                  " [        S9n U R                  S[        SS9  U R                  S[        [	        S5      S9  U R                  5       n[        UR                  5      n[        U5      u  p4UR                  R                  R                  SSS9  UR                  R                  [        R                  " US	S
95        UR                  R                  S-  R                  [        R                  " US	S
95        U Vs/ s H  oUS   S:X  d  M  UPM     nnU Vs/ s H  oUS   S:”  d  M  UPM     nn[        R                   " S[#        U5      [#        U5      [#        U5      -
  [#        U5      [#        U5      S9  U(       a  [        R$                  " S[#        U5      S9  U H&  n[        R$                  " SUS   US   S S US   S9  M(     g s  snf s  snf )N)Údescriptionz	--journalT)r   Úrequiredz--outzoutput/benchmark/gold.json)r   Údefault)ÚparentsÚexist_oké   )Úindentzgold_dropped.jsonrN   rG   rO   é   z:kept {v} value + {p} prose from {d} documents; dropped {x})ÚvÚpÚdÚxzI{n} kept value questions have a gold figure printed in more than one cell)Únzdropped [{s}] {q!r}: {w}rB   rH   é<   rF   )r   rW   Úw)ÚargparseÚArgumentParserÚ__doc__Úadd_argumentr   Ú
parse_argsr&   r"   rY   ÚoutÚparentÚmkdirÚ
write_textr   Údumpsr   Úsuccessr;   Úwarning)	rd   Úargsr#   rT   rU   ÚkÚvaluesÚrepeatedre   s	            r   Úmainrz   …   sŠ  € Ü×Ò¬GÑ4€AØ‡N�N�;¤T°D€NÑ9Ø‡N�N�7¤¬tÐ4PÓ/Q€NÑRØ�<‰<‹>€Dä˜$Ÿ,™,Ó'€DÜ˜4“L�M€DØ‡H�H‡O�O×Ñ $°ÐÑ6Ø‡H�H×ÑœŸ
š
 4°Ñ2Ô3Ø	‡X�X‡_�_Ð*Ñ*×6Ñ6´t·z²zÀ'ÐRSÑ7TÔUáÓ6š�A 6¡¨gÑ!5�a™€FÐ6Ù!ÓFš6�aÐ'@Ñ%AÀAÑ%E—™6€HÐFÜ
‡N‚NØDÜ
ˆf‹+Ü
ˆd‹)”c˜&“kÑ
!Ü
ˆd‹)Ü
ˆg‹,òö Ü�ŠØWÜ�(‹mò	
ó ˆÜ�ŠÐ1°Q°v±YÀ!ÀJÁ-ÐPSÐQSÐBTÐXYÐZ_ÑX`Ôaò ùò 7ùÚFs   ÄGÄGÄ+G"Ä;G"Ú__main__)r   ÚstrÚreturnr|   )r"   r   r}   úList[Dict[str, Any]])r*   r|   r}   zList[Tuple[str, str]])r3   r|   r}   úDict[Tuple[str, str], str])r6   r   r=   r|   r}   Úint)r#   r~   rS   z+Callable[[str], Dict[Tuple[str, str], str]]r}   z1Tuple[List[Dict[str, Any]], List[Dict[str, Any]]])r}   ÚNone)rl   Ú
__future__r   rj   r   r   Úpathlibr   Útypingr   r   r   r   r	   r
   Úlogurur   Úquber.playgroundr   r   r&   r-   r9   r@   rY   rz   Ú__name__© r   r   Ú<module>r‰      s�   ðñõ  #ã Û Û 	Ý ß =× =å å ô2ôôôô"
^ð >Hð1Ø
ð1à:ð1ð 7õ1ôhbð< ˆzÓÙ…Fð r   