ó
    á³j  ã                  óä   • S r SSKJr  SSKrSSKJs  Jr  SSK	r	SSK
Jr  \	R                  R                  r\	R                  " SS9S 5       rS rS	 r\	R                  R&                  S
 5       rS rg)a”  Integration against the development Postgres and its ingested corpus. A
mocked query proves the mock, not the SQL, so these run the real statements:
reference resolution over the groundings table, the fused retrieval window
(which embeds the query, so the local embedding model loads), and the cell
index gold verification depends on. Skipped when the database is unreachable
or holds no fusion document.é    )ÚannotationsN)ÚdbÚmodule)Úscopec                 ó>  •  [         R                  " 5        n U R                  S5      R                  5       nSSS5        W(       d  [
        R                  " S5        U$ ! , (       d  f       N-= f! [         a#  n[
        R                  " SU 35         SnANXSnAff = f)z.(id, doc_key) of one ingested fusion document.zlSELECT id, doc_key FROM ade_playground.documents WHERE ade_version = 'quber-fusion' ORDER BY doc_key LIMIT 1Nz"development Postgres unreachable: zno fusion document ingested)r   ÚconnectÚexecuteÚfetchoneÚ	ExceptionÚpytestÚskip)ÚconnÚrowÚexcs      Ú>/home/mande/repo/quber/tests/playground/test_db_integration.pyÚ
fusion_docr      s�   € ð@Ü�ZŠZŒ\˜TØ—,‘,ðNó÷ ‰h‹jð ÷ ö Ü�ŠÐ1Ô2Ø€J÷ �\ûô
 ó @Ü�ŠÐ8¸¸Ð>×?Ñ?ûð@ús3   ‚A/ — A·A/ Á
A,Á(A/ Á,A/ Á/
BÁ9BÂBc                óª  • SSK Jn  U u  p#[        R                  " 5        nUR	                  SU45      R                  5       nS S S 5        W(       d‡  [        R                  " S5      S-   SS[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS0-  n[        [        R                  " U5      5      eU Vs/ s H  owS   PM	     nnU" X(5      n	U	 Vs/ s H  owR                  PM     n
nX¨:H  o»(       d¥  [        R                  " SU4S	X¨45      [        R                  " U
5      S
[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS
S.-  nSSU0-  n[        [        R                  " U5      5      eS =p«U	S   nUR                   n/ nUS   S   nUnU(       d  SnUnSnUU-   nUU:H  nU(       Gd5  SS[        R                  " U5      0-  nUR#                  U5        U(       d,  SS[        R                  " W5      0-  nUR#                  U5        [        R$                  " US5      0 -  n[        R                  " SU4SUU45      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      U[        R                  " U5      S.-  nSSU0-  n[        [        R                  " U5      5      eS =n=n=n=n=n=n=nnUR&                  oõS   S   nUU:H  nU(       d»  [        R                  " SU4SUU45      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      [        R                  " U5      S.-  nSSU0-  n[        [        R                  " U5      5      eS =n=nnUR(                  nS nUULnU(       d»  [        R                  " SU4SUU45      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      [        R                  " U5      S.-  nSSU0-  n[        [        R                  " U5      5      eS =n=nnS/o±" X+5      n/ nUU:H  nU(       Gd  [        R                  " SU4SUU45      S [        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS S![        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS![        R                  " U5      [        R                  " U5      [        R                  " U5      S".-  nS#SU0-  n[        [        R                  " U5      5      eS =n=n=nng ! , (       d  f       GNþ= fs  snf s  snf )$Nr   ©Ú_resolve_refszÉSELECT g.ref_id, g.page, g.status FROM ade_playground.groundings g
               WHERE g.document_id = %s AND g.ref_type = 'tableCell' AND g.status IS NOT NULL
               ORDER BY g.ref_id LIMIT 2z3expected at least one grounded cell with provenanceú
>assert %(py0)sÚpy0r   ©ú==)z%(py1)s == %(py3)sÚcited)Úpy1Úpy3zassert %(py5)sÚpy5é   z%(py7)sÚpy7z%(py10)sÚpy10)z:%(py2)s
{%(py2)s = %(py0)s.page
} == (%(py13)s + %(py15)s)Úfirst)r   Úpy2Úpy13Úpy15zassert %(py18)sÚpy18é   )z.%(py2)s
{%(py2)s = %(py0)s.status
} == %(py5)s)r   r"   r   zassert %(py7)s©úis not)z0%(py2)s
{%(py2)s = %(py0)s.text
} is not %(py5)sz
no-such-id)z9%(py5)s
{%(py5)s = %(py0)s(%(py1)s, %(py3)s)
} == %(py8)sr   Údoc_id)r   r   r   r   Úpy8zassert %(py10)s)Úquber.playground.appr   r   r   r	   ÚfetchallÚ
@pytest_arÚ_format_assertmsgÚ@py_builtinsÚlocalsÚ_should_repr_global_nameÚ	_safereprÚAssertionErrorÚ_format_explanationÚref_idÚ_call_reprcompareÚpageÚappendÚ_format_boolopÚstatusÚtext)r   r   r)   Ú_r   r   Ú@py_format1Úrr   ÚrefsÚ@py_assert0Ú@py_assert2Ú@py_format4Ú@py_format6r!   Ú@py_assert1Ú@py_assert5Ú@py_assert6Ú@py_assert4Ú@py_assert9Ú@py_assert14Ú@py_assert16Ú@py_assert3Ú@py_format8Ú@py_format11Ú@py_format12Ú@py_format17Ú@py_format19Ú@py_assert7Ú@py_format9s                                 r   Ú1test_reference_resolution_against_known_cited_idsrS   !   sš  € Ý2à�I€FÜ	�ŠŒ˜Ø�l‰lð,ð ˆIó	
÷
 ‰(‹*ð 	÷ 
ö ×E×EÐE×E×E×E×E×E×EÐEˆ3×E×E×EÑEˆ3×E×E×E×E×EÐEÙÓš3�aˆqŒT™3€EÐá˜Ó'€DÙ"Ó#šd˜�HŒH™dÐ,Ñ#Ñ,×,×,×,Ò,Ð#×,×,Ô,Ð#×,×,×,×,×,Ô, u×,×,×,Ñ, u×,×,×,×,×,×,Ó,Ø�‰G€EØ�:‰:Ò-˜#˜a™& ™)Ð-˜)Ð-ž) qÐ- qÐ-¨AÑ-¨AÑ-Ð-ˆ:Ð-Ñ-×-×-×-Ò-˜)×-×-×-Ð-ž)×-×-Ñ- q×-×-×-×-×-×-×-×-Ö-ˆ:Ð-×-×-×-×-×-Õ-ˆ5×-×-×-Ñ-ˆ5×-×-Õ-ˆ:×-×-Ô-¨A×-×-×-×-×-×-×-Ö-Ø�<‰<Ð$˜q™6 !™9Ð$ˆ<˜9Ñ$×$×$×$Ó$ˆ<˜9×$×$×$×$×$Õ$ˆ5×$×$×$Ñ$ˆ5×$×$Õ$ˆ<×$×$Ó$˜9×$×$×$×$×$×$Ô$Ø�:‰:Ð!˜TÐ!ˆ:˜TÐ!×!×!×!Ó!ˆ:˜T×!×!×!×!×!Õ!ˆ5×!×!×!Ñ!ˆ5×!×!Õ!ˆ:×!×!Ó!˜T×!×!×!×!×!×!Ô!ð #/ Ð6ˆ=˜Ó0Ð6°BÐ6Ð0°BÑ6×6×6×6Ô6Ð0°B×6×6×6×6×6Õ6ˆ=×6×6×6Ñ6ˆ=×6×6×6×6×6Ö6˜×6×6×6Ñ6˜×6×6Õ6 ×6×6Ó6Ð0×6×6Ó6°B×6×6×6×6×6×6×6÷% 
Žüò  ùò $s    "X9ÃYÃ8YØ9
Yc                ó¢  • SSK Jn  U u  p#[        R                  " 5        nUR	                  SU45      R                  5       nS S S 5        W(       d  [        R                  " S5        Uu  pgU" X'U/5       Vs/ s H  oˆR                  PM     n	n/ o¦U	;   o»oË(       a  Xy;  oÝnU(       Gdà  [        R                  " SU4SXi45      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OSS[        R                  " 5       ;   d  [        R                  " U	5      (       a  [        R                  " U	5      OSS	.-  nS
SU0-  oúR                  U5        U(       aÍ  [        R                  " SW4SXy45      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OSS[        R                  " 5       ;   d  [        R                  " U	5      (       a  [        R                  " U	5      OSS.-  nSSU0-  nU
R                  U5        [        R                   " U
S5      0 -  nSSU0-  n[#        [        R$                  " U5      5      eS =n=n
=p½U" X'/5       Vs/ s H  oˆR                  PM     nnU/o¼U:H  nU(       do  [        R                  " SU4SXË45      [        R                  " U5      [        R                  " U5      S.-  nSSU0-  n[#        [        R$                  " U5      5      eS =n=nng ! , (       d  f       GNú= fs  snf s  snf )Nr   r   a«  SELECT c.ref_id, l.ref_id
               FROM ade_playground.groundings c
               JOIN ade_playground.groundings l
                 ON l.document_id = c.document_id
                AND l.ref_type = 'line_item'
                AND l.ref_id = split_part(c.ref_id, '-', 1) || '-line-' || (c.position->>'row')
               WHERE c.document_id = %s AND c.ref_type = 'tableCell' AND c.bbox IS NOT NULL
               LIMIT 1z%no cell with an enclosing line record)Úin)z%(py2)s in %(py4)sÚcell_idÚids)r"   Úpy4z%(py6)sÚpy6)únot in)z%(py8)s not in %(py10)sÚline_id)r*   r    z%(py12)sÚpy12zassert %(py15)sr$   r   )z%(py1)s == %(py4)s©r   rX   úassert %(py6)s)r+   r   r   r   r	   r
   r   r   r5   r-   r6   r/   r0   r1   r2   r8   r9   r3   r4   )r   r   r)   r<   r   r   rV   r[   r>   rW   rD   rK   r@   rH   Ú@py_format5Ú@py_format7rM   Ú@py_format13Ú@py_format14Ú@py_format16rA   s                        r   Ú*test_cited_cell_suppresses_its_line_recordrd   :   sã  € Ý2à�I€FÜ	�ŠŒ˜Ø�l‰lðð ˆIó

÷ ‰(‹*ð 	÷ 
ö Ü�ŠÐ;Ô<ØÑ€Gá*¨6¸WÐ3EÔFÓ
GÒF˜�8Œ8ÑF€CÐ
GÙ0�c‰>×0˜gÑ0×0×0×0Õ0ˆ7×0×0×0×0×0Õ0ˆ7×0×0×0Ñ0ˆ7×0×0×0×0×0Ö0�c×0×0×0Ñ0�c×0×0×0×0×0×0×0Ô0˜g×0×0×0×0×0Õ0˜g×0×0×0Ñ0˜g×0×0×0×0×0Ö0¨S×0×0×0Ñ0¨S×0×0×0×0×0×0×0×0×0×0×0×0×0Ñ0ñ ,¨F°IÔ>Ó?Ò>˜�HŒHÑ>ÐLÐ?ÀGÀ9ÐLÀ9ÑL×L×L×LÓLÐ?×L×LÔLÐ?×L×LÓLÀ9×L×L×L×L×L×LÕL÷) 
Žüò  Hùò @s    "L5Á7MÊMÌ5
Mc           	     ó~
  • SS K nSSKJnJn  U u  pEUR	                  U" USS5      5      nU(       d‡  [
        R                  " S5      S-   SS[        R                  " 5       ;   d  [
        R                  " U5      (       a  [
        R                  " U5      OS0-  n[        [
        R                  " U5      5      eU Vs/ s H  oˆR                  PM     n	nS	n
[        XšS
9o¹U:H  oÌ(       GdX  [
        R                  " SU4SX›45      S[        R                  " 5       ;   d  [
        R                  " U	5      (       a  [
        R                  " U	5      OSS[        R                  " 5       ;   d  [
        R                  " [        5      (       a  [
        R                  " [        5      OSS[        R                  " 5       ;   d  [
        R                  " U	5      (       a  [
        R                  " U	5      OS[
        R                  " U
5      [
        R                  " U5      S.-  nSSU0-  n[        [
        R                  " U5      5      eS =n=p«U Vs1 s H  oˆR                   iM     nn[#        U5      n[#        U5      nUU:H  nU(       Gdv  [
        R                  " SU4SUU45      S[        R                  " 5       ;   d  [
        R                  " ["        5      (       a  [
        R                  " ["        5      OS[
        R                  " U5      [
        R                  " U5      S[        R                  " 5       ;   d  [
        R                  " ["        5      (       a  [
        R                  " ["        5      OSS[        R                  " 5       ;   d  [
        R                  " U5      (       a  [
        R                  " U5      OS[
        R                  " U5      S.-  nSSU0-  n[        [
        R                  " U5      5      eS =n=n=nnU" US5      n0 nU HT  nUR$                  S:X  d  M  UR&                  (       d  M(  UR)                  UR&                  S5      S-   UUR&                  '   MV     S UR+                  5        5       n[-        U5      oÿ(       d¢  SS[        R                  " 5       ;   d  [
        R                  " [,        5      (       a  [
        R                  " [,        5      OS[
        R                  " U5      [
        R                  " U5      S.-  n[        [
        R                  " U5      5      eS =pÏg s  snf s  snf )Nr   )Ú_cap_line_recordsÚ_fused_windowztotal revenue for the quarteré   z7fused window returned nothing over an ingested documentr   r   ÚwindowT)Úreverser   )zA%(py0)s == %(py7)s
{%(py7)s = %(py2)s(%(py3)s, reverse=%(py5)s)
}ÚscoresÚsorted)r   r"   r   r   r   úassert %(py9)sÚpy9)zN%(py4)s
{%(py4)s = %(py0)s(%(py2)s)
} == %(py9)s
{%(py9)s = %(py6)s(%(py7)s)
}Úlen)r   r"   rX   rY   r   rn   zassert %(py11)sÚpy11r&   Ú	line_itemr   c              3  ó*   #   • U  H	  oS :*  v •  M     g7f)r&   N© )Ú.0Úns     r   Ú	<genexpr>Ú9test_fused_window_over_the_real_corpus.<locals>.<genexpr>f   s   é € Ð3Ò2˜!�AŽvÒ2ùs   ‚z,assert %(py4)s
{%(py4)s = %(py0)s(%(py2)s)
}Úall)r   r"   rX   )ÚasyncioÚquber.playground.retrievalrf   rg   Úrunr-   r.   r/   r0   r1   r2   r3   r4   Úscorerl   r6   Úchunk_idro   Ú
chunk_typeÚparent_chunk_idÚgetÚvaluesrx   )r   ry   rf   rg   r<   Údoc_keyri   r=   Úcrk   rG   rF   rD   rL   Ú@py_format10rK   Ú@py_assert8rE   rN   ÚcappedÚ
per_parentr_   s                         r   Ú&test_fused_window_over_the_real_corpusrˆ   U   sê  € ãçKà�J€AØ�[‰[™ wÐ0OÐQSÓTÓU€FÞ×L×LÐL×L×L×L×L×L×LÐLˆ6×L×L×LÑLˆ6×L×L×L×L×LÐLÙ%Ó&šv˜!�gŒg™v€FÐ&Ø,0Ð1”V˜FÑ1Ð1Ð1Ñ1×1×1×1Ó1ˆ6×1×1×1×1×1Õ1ˆ6×1×1×1Ñ1ˆ6×1×1×1×1×1Ö1”V×1×1×1Ñ1”V×1×1×1×1×1Ö1˜F×1×1×1Ñ1˜F×1×1Õ1¨D×1×1Ó1Ð1×1×1×1×1×1×1Ó1Ù$*Ó+¢F˜q—
”
¡FÐ;Ð+Œ3Ð+Ó,Ð;´°F³Ð;Ð,°Ñ;×;×;×;Ô;Ð,°×;×;×;×;×;Õ;Œ3×;×;×;Ñ;Œ3×;×;Õ;Ð+×;×;Ó;Ð,×;×;×;×;×;Ô;´×;×;×;Ñ;´×;×;×;×;×;Ö;°F×;×;×;Ñ;°F×;×;Õ;°×;×;×;×;×;×;Ö;Ù˜v qÓ)€FØ€JÛˆØ�<‰<˜;Õ&¨1×+<×+<Ñ+<Ø,6¯N©N¸1×;LÑ;LÈaÓ,PÐSTÑ,TˆJ�q×(Ñ(Ó)ñ ñ 4˜z×0Ñ0Ô2Ó3Ð3Œ3Ð3Ó3Ð3Õ3×3×3×3×3×3Ñ3Œ3×3×3×3Ñ3Œ3×3×3Õ3Ð3×3×3Ó3Ð3×3×3×3×3×3Ó3ùò 'ùâ+s   Â<T5É
T:c                óˆ  • SSK JnJn  U u  p4U" U5      nU(       d‡  [        R                  " S5      S-   SS[
        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS0-  n[        [        R                  " U5      5      e[        [        UR                  5       5      5      u  u  p7nUSU4   n	S o©U
Lo»(       do  [        R                  " SU4S	Xš45      [        R                  " U	5      [        R                  " U
5      S
.-  nSSU0-  n[        [        R                  " U5      5      eS =n	=pºU(       Gak  U" XX5      n
SoêU:¬  oÿ(       GdP  [        R                  " SU4SX®45      S[
        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OSS[
        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OSS[
        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U
5      [        R                  " U5      S.-  nSSU0-  n[        [        R                  " U5      5      eS =n
=pþg g )Nr   )Ú
cell_indexÚoccurrencesz4expected tagged cells in an ingested fusion documentr   r   ÚindexÚ r'   )z%(py1)s is not %(py4)sr]   r^   rY   r   )ú>=)z9%(py4)s
{%(py4)s = %(py0)s(%(py1)s, %(py2)s)
} >= %(py7)sr‹   r;   )r   r   r"   rX   r   rm   rn   )Ú&quber.playground.benchmark.verify_goldrŠ   r‹   r-   r.   r/   r0   r1   r2   r3   r4   ÚnextÚiterÚitemsr6   )r   rŠ   r‹   r<   r‚   rŒ   r=   Úcidr;   r@   rK   rA   r_   r`   rF   rE   rL   r„   s                     r   Ú$test_cell_index_over_the_real_corpusr”   i   s†  € ßNà�J€AÙ�wÓ€EÞ×H×HÐH×H×H×H×H×H×HÐHˆ5×H×H×HÑHˆ5×H×H×H×H×HÐHÜœ$˜uŸ{™{›}Ó-Ó.�N�H€Qˆdà�"�c�ÑÐ' 4Ð' 4Ð'×'×'×'Ò'Ð×'×'Ô'Ð×'×'Ó' 4×'×'×'×'×'×'Ó'ßÙ˜5Ó'Ð,¨1Ð,¨1Ñ,×,×,×,Ó,Ð'×,×,×,×,×,Õ,ˆ{×,×,×,Ñ,ˆ{×,×,×,×,×,Ö,˜5×,×,×,Ñ,˜5×,×,×,×,×,Ö, $×,×,×,Ñ, $×,×,Õ,Ð'×,×,Ó,¨1×,×,×,×,×,×,Ô,ð ó    )Ú__doc__Ú
__future__r   Úbuiltinsr/   Ú_pytest.assertion.rewriteÚ	assertionÚrewriter-   r   Úquber.playgroundr   ÚmarkÚintegrationÚ
pytestmarkÚfixturer   rS   rd   Úslowrˆ   r”   rs   r•   r   Ú<module>r¢      ss   ðñ õ #ç  „ ƒå à�[‰[×$Ñ$€
ð ‡‚�hÑñó  ðò7ò2Mð6 ‡�×Ññ4ó ð4ó&
-r•   