ó
    6ljd  ã                  ó6  • S r SSKJr  SSKrSSKrSSKrSSKrSSKJr  SSK	J
r
  SSKJr  SSKJr  SSKJr  SS	KJr  \" 5       R&                  R(                  r1 S
kr\R.                  " S\R0                  5      rSS jrSSS jjrSSS jjr\S:X  a  \" 5         gg)ad  Ingest one parsed ADE document into the pgvector playground schema.

Input: an ADE parse JSON (the `.ade.json` produced by Landing.ai) and the
source PDF. Both may be local paths or `s3://` URIs.

Steps:
  1. Stage the source PDF under data/<doc_key>.pdf so the app can serve it.
  2. Insert the document row.
  3. Insert every chunk; embed text/table chunk markdown into pgvector.
  4. Insert every grounding (chunk boxes and individual table cells) so the
     app can resolve any cited id to a page + bounding box overlay.

The ADE JSON shape (verified against real extractions):
  { markdown, chunks: [ {markdown, type, id, grounding:{box,page}} ],
    grounding: { <id>: {box, page, type, position?} }, metadata: {...} }
where box is {left, top, right, bottom} normalized to 0..1, and cell
grounding entries carry position {row, col, rowspan, colspan, chunk_id}.
é    )ÚannotationsN)ÚPath)ÚOptional)Úlogger)Údb)Úembed_texts)Úget_settings>   ÚtextÚtableÚ
marginaliaz<a[^>]*></a>\s*c           	     ó,  • UR                   R                  SSS9  U R                  S5      (       a;  [        R                  " SXS9  [
        R                  " SSSU [        U5      /SSS	9  U$ UR                  [        U 5      R                  5       5        U$ )
z5Copy a local path or s3:// URI to dest. Returns dest.T)ÚparentsÚexist_okzs3://zDownloading {src} -> {dest}©ÚsrcÚdestÚawsÚs3Úcp)ÚcheckÚcapture_output)ÚparentÚmkdirÚ
startswithr   ÚinfoÚ
subprocessÚrunÚstrÚwrite_bytesr   Ú
read_bytesr   s     Ú5/home/mande/repo/quber/src/quber/playground/ingest.pyÚ
fetch_filer"   0   s€   € à‡K�K×Ñ˜d¨TÐÑ2Ø
‡~�~�g×ÑÜ�ŠÐ1°sÒFÜ�Š˜˜t T¨3´°D³	Ð:À$ÐW[Ò\ð €Kð 	×Ñœ˜c›×-Ñ-Ó/Ô0Ø€Kó    c                óì  • [        U[        U  S3-  5      n[        U[        U  S3-  5      n[        R                  " UR	                  5       5      nUS   nU HJ  n[
        R                  SUR                  SS5      5      R                  SS5      R                  5       US'   ML     UR                  S0 5      n	UR                  S0 5      n
U Vs/ s HF  oˆR                  S	5      [        ;   d  M  UR                  SS5      R                  5       (       d  MD  UPMH     nnU(       a  [        U Vs/ s H  oˆS   PM	     sn5      O/ n[        X¼S
S9 VVs0 s H  u  p�US   U_M     nnn[        R                  " S[        U5      U S9  [         R"                  " UR%                  5       5      R'                  5       n[(        R*                  " 5        nUR-                  SU 45        UR-                  SU UU=(       d    U
R                  SU  S35      U
R                  S5      U
R                  S5      45      R/                  5       nUc   eUS   nU Hœ  nUR                  S5      =(       d    0 nUR-                  SUUS   UR                  S	5      UR                  S5      [        R0                  " UR                  S5      5      UR                  SS5      UR                  US   5      45        Mž     U	R3                  5        Hš  u  nnUR-                  SUUUR                  S	5      UR                  S5      [        R0                  " UR                  S5      5      UR                  S5      (       a%  [        R0                  " UR                  S5      5      OS45        Mœ     SSS5        [        R4                  " SU [        U5      [        U	5      S9  [        U5      $ s  snf s  snf s  snnf ! , (       d  f       NR= f)a  Ingest one document. Returns the number of embedded chunks.

`doc_key` is the opaque storage key: it names the staged files and the
document row, and is never displayed. `filename` is the original PDF
filename kept for presentation; absent, the ADE metadata's name stands.
z.pdfz	.ade.jsonÚchunksÚ ÚmarkdownÚ Ú	groundingÚmetadataÚtypeT)ÚstrictÚidzEmbedded {n} chunks for {key})ÚnÚkeyz7DELETE FROM ade_playground.documents WHERE doc_key = %sz¦INSERT INTO ade_playground.documents
               (doc_key, content_hash, filename, page_count, ade_version)
               VALUES (%s, %s, %s, %s, %s) RETURNING idÚfilenameÚ
page_countÚversionNr   z¯INSERT INTO ade_playground.chunks
                   (document_id, chunk_id, chunk_type, page, bbox, content, embedding)
                   VALUES (%s, %s, %s, %s, %s, %s, %s)ÚpageÚboxzáINSERT INTO ade_playground.groundings
                   (document_id, ref_id, ref_type, page, bbox, position)
                   VALUES (%s, %s, %s, %s, %s, %s)
                   ON CONFLICT (document_id, ref_id) DO NOTHINGÚpositionz*Ingested {key}: {c} chunks, {g} groundings)r/   ÚcÚg)r"   ÚDATA_DIRÚjsonÚloadsÚ	read_textÚ
_ANCHOR_REÚsubÚgetÚreplaceÚstripÚEMBEDDABLE_TYPESr   Úzipr   r   ÚlenÚhashlibÚsha256r    Ú	hexdigestr   ÚconnectÚexecuteÚfetchoneÚdumpsÚitemsÚsuccess)Údoc_keyÚade_jsonÚ
source_pdfr0   Ú	pdf_localÚ	ade_localÚdocr%   r6   r)   ÚmetaÚto_embedÚvectorsÚvÚ	vec_by_idÚcontent_hashÚconnÚrowÚdoc_idr7   Úref_idÚentrys                         r!   Úingestr^   ;   s]  € ô ˜:¤x°W°I¸TÐ2BÑ'BÓC€IÜ˜8¤X°7°)¸9Ð0EÑ%EÓF€IÜ
�*Š*�Y×(Ñ(Ó*Ó
+€CØ�‰]€FÛˆô #Ÿ™ r¨1¯5©5°¸RÓ+@ÓA×IÑIÈ&ÐRTÓU×[Ñ[Ó]ˆˆ*‹ñ ð —‘˜ RÓ(€IØ�7‰7�:˜rÓ"€Dñ "Óiš6�a§U¡U¨6£]Ô6FÑ%F“È1Ï5É5ÐQ[Ð]_ÓK`×KfÑKf×Kh—™6€HÐiÞ@HŒk±(Ó;²(¨Q˜Zœ=±(Ñ;Ô<Èb€GÜ(+¨HÀdÒ(KÔLÒ(K¡ ��4‘˜!’Ñ(K€IÑLÜ
‡K‚KÐ/´3°w³<ÀWÒMä—>’> )×"6Ñ"6Ó"8Ó9×CÑCÓE€LÜ	�ŠŒ˜à�‰ÐNÐQXÐPZÔ[Ø�l‰lð;ð ØØ×B˜DŸH™H Z°G°9¸DÐ1AÓBØ—‘˜Ó&Ø—‘˜Ó#ðó	
÷ ‰(‹*ð 	ð ‰ÐˆØ�Q‘ˆó ˆAØ—‘�kÓ"×( bˆAØ�L‰Lð:ð Ø�d‘GØ—E‘E˜&“MØ—E‘E˜&“MÜ—J’J˜qŸu™u U›|Ó,Ø—E‘E˜* bÓ)Ø—M‘M ! D¡'Ó*ðö	ñ ð$ 'Ÿ_™_Ö.‰MˆF�EØ�L‰LðCð
 ØØ—I‘I˜fÓ%Ø—I‘I˜fÓ%Ü—J’J˜uŸy™y¨Ó/Ó0Ø9>¿¹À:×9NÑ9N”D—J’J˜uŸy™y¨Ó4Ô5ÐTXðöñ /÷K 
ôj ‡N‚NÐ?ÀWÔPSÐTZÓP[Ô_bÐclÓ_mÒnÜˆw‹<Ðùòy jùÚ;ùÛL÷ 
�ús+   ÃOÃ.#OÄOÄ-OÅOÇGO%Ï%
O3c                óÖ  • SS K nUR                  SS9nUR                  SSSS9  UR                  SSS	S9  UR                  S
SSS9  UR                  SS SS9  UR                  SSSS9  UR                  U 5      nUR                  (       a+  [
        R                  " 5         [        R                  " S5        [        UR                  UR                  UR                  UR                  S9  g )Nr   z;Ingest an ADE document into the playground pgvector schema.)Údescriptionz	--doc-keyTz#opaque storage key for the document)ÚrequiredÚhelpz
--ade-jsonz(local path or s3:// URI to the .ade.jsonz--pdfz)local path or s3:// URI to the source PDFz
--filenamez,original PDF filename, kept for presentation)Údefaultrb   z--init-schemaÚ
store_truez#(re)create the schema before ingest)Úactionrb   zApplied schema)r0   )ÚargparseÚArgumentParserÚadd_argumentÚ
parse_argsÚinit_schemar   Úapply_schemar   rL   r^   rM   rN   Úpdfr0   )Úargvrf   ÚpÚargss       r!   Úmainrp   Ž   sÄ   € Ûà×ÑÐ,iÐÐj€AØ‡N�N�;¨Ð4Y€NÑZØ‡N�N�<¨$Ð5_€NÑ`Ø‡N�N�7 TÐ0[€NÑ\Ø‡N�N�<¨Ð4b€NÑcØ‡N�N�?¨<Ð>c€NÑdØ�<‰<˜Ó€Dà××Ü
�ŠÔÜ�ŠÐ'Ô(Ü
ˆ4�<‰<˜Ÿ™¨¯©¸4¿=¹=ÓIr#   Ú__main__)r   r   r   r   Úreturnr   )N)
rM   r   rN   r   rO   r   r0   zOptional[str]rr   Úint)rm   zOptional[list[str]]rr   ÚNone)Ú__doc__Ú
__future__r   rD   r9   Úrer   Úpathlibr   Útypingr   Úlogurur   Úquber.playgroundr   Úquber.playground.embeddingr   Úquber.settingsr	   Ú
playgroundÚdata_dirr8   rA   ÚcompileÚ
IGNORECASEr<   r"   r^   rp   Ú__name__© r#   r!   Ú<module>r„      s…   ðñõ& #ã Û Û 	Û Ý Ý å å Ý 2Ý 'á‹>×$Ñ$×-Ñ-€ò 3Ð ð �ZŠZÐ*¨B¯M©MÓ:€
ôöPöfJð" ˆzÓÙ…Fð r#   