ó
    XI=j )  ã                  óâ  • S r SSKJr  SSKrSSKrSSKrSSKrSSKJr  SSK	J
r
JrJr  SSKrSSKJr  \R                   R#                  S\" \" \5      R)                  5       R*                  5      5        SSKrSSKJr  SSKJrJrJrJrJrJ r J!r!J"r"J#r#J$r$  SS	K%J&r&J'r'  \" \5      R)                  5       RP                  S
   r)\)S-  r*Sr+Sr,Sr-Sr.\" \5      R)                  5       R*                  S-  S-  r/S!S jr0Sr1Sr2S"S jr3            S#S jr4            S$S jr5          S%S jr6S&S jr7S'S jr8S(S jr9S)S jr:\;S:X  aŸ  \Rx                  SS r<S\<;   a!  \<R{                  S5      r>\<\>S-      r.\<\>\>S
-   2	 \" \5      R)                  5       R*                  S-  S \. 3-  r/\<(       a  \<O\3" 5        V s/ s H  n \" U 5      PM     sn r?\R€                  " \:" \?5      5        ggs  sn f )*aù  QUE-270: draw table (SoM) and footnote regions as distinct boxes on a corpus.

Goal
----
Expand the region identifiers so every page carries TWO kinds of box: the
grid-locator's table regions (blue) and a dedicated footnote finder's grounded
footnote regions (orange). Together they delimit the combined table+footnote
target a downstream extraction agent would read. This script produces an
annotated PDF per document plus a JSON index, run across the whole real corpus,
so the boxes can be eyeballed at scale rather than reasoned about on a few pages.

It reuses, unchanged:
  - PydanticAIGridLocator for table regions (the production locator);
  - the dedicated footnote-finder prompt/schema and the marker-grounded region
    logic validated in footnote_finder_probe.py;
  - PyMuPDF box drawing, mirroring src/quber/review/annotated.py.

Run
---
    uv run python experiments/que270/annotate_regions.py            # whole corpus
    uv run python experiments/que270/annotate_regions.py <pdf> ...  # specific docs
é    )ÚannotationsN)ÚPath)ÚListÚOptionalÚTuple)Úlogger)Ú
page_words)
ÚBOX_PAD_PTSÚDEFAULT_COLSÚDEFAULT_ROWSÚGridFlagÚLocatedTableÚPydanticAIGridLocatorÚdeclash_stackedÚflags_to_locatedÚoverlay_gridÚ	pad_boxes)Úinit_s3_cacheÚresolve_documenté   z.cache/s3/qubera-docséÈ   é   é<   ÚprodÚoutÚ	annotatedc                ó€  • [         R                  " [        U 5      5      n[        UR                  U5      nUS-  n[         R
                  " Xf5      n/ n[        U5       HL  n	XI   R                  US9n
USU	S-   S S3-  nU
R                  [        U5      5        UR                  U5        MN     UR                  5         U$ )z‰Render the first `cap` pages of `pdf` to PNGs via PyMuPDF (honors the cap
cheaply; a 500-page filing renders only its first `cap` pages).g      R@)Úmatrixzpage-é   Ú04dz.png)ÚfitzÚopenÚstrÚminÚ
page_countÚMatrixÚrangeÚ
get_pixmapÚsaveÚappendÚclose)ÚpdfÚout_dirÚdpiÚcapÚdocÚnÚzoomÚmatÚpathsÚiÚpixÚpaths               Ú=/home/mande/repo/quber/experiments/que270/annotate_regions.pyÚ_render_cappedr9   C   sª   € ô �)Š)”C˜“HÓ
€CÜˆC�N‰N˜CÓ €AØ�‰:€DÜ
�+Š+�dÓ
!€CØ€EÜ�1ŽXˆØ‰f×Ñ sÐÐ+ˆØ˜5  Q¡ s ¨4Ð0Ñ0ˆØ�‰”�T“ÔØ�‰�TÖñ	 ð
 ‡I�I„KØ€Ló    )ç        g{®GázÔ?gq=
×£pí?)gffffffî?gÍÌÌÌÌÌÜ?r;   c                 óL   • [        S [        R                  S5       5       5      $ )zCEvery real (non-synthetic) PDF under the qubera-docs cache, sorted.c              3  ó$   #   • U  H  ov •  M     g 7f©N© )Ú.0Úps     r8   Ú	<genexpr>Úcorpus.<locals>.<genexpr>Y   s   é € Ð1Ò0˜”!Ò0ùs   ‚z*.pdf)ÚsortedÚDOCSÚrglobr?   r:   r8   ÚcorpusrG   W   s   € äÑ1œTŸZ™Z¨Ô0Ó1Ó1Ð1r:   c              ƒ  ó   #   • SSK Jn  U R                  XBU5      I Sh  v•N nU Vs/ s H%  owR                  UR                  UR
                  4PM'     nn/ n	U(       aÒ  [        R                  " [        X#5      I Sh  v•N u  p«n[        R                  " SS9 n[        U5      S-  n[        R                  " [        XN[        [        5      I Sh  v•N   U" UR                  5       SS9n UR!                  S	U/5      I Sh  v•N nUR"                  R$                  nSSS5        [/        WXlX«5      n	X8U	4$  GNs  snf  N¿ Nn N?! [&         a.  n[(        R*                  " S
X2R,                  U5        / n SnANZSnAff = f! , (       d  f       Nh= f7f)a  Locate tables and footnote blocks on one page; return both as norm boxes.

Tables come from the production locator. Footnote blocks come from the
dedicated finder, each grounded to a precise region off the PDF text layer.
Footnotes are only sought on pages that have at least one table.
r   )ÚBinaryContentNz
que270-fn-©Úprefixzgridded.pngz	image/png)ÚdataÚ
media_typezReport using the grid.z(footnote finder failed page={} of {}: {})Úpydantic_airI   ÚlocateÚordinalÚtitleÚregionÚasyncioÚ	to_threadr	   ÚtempfileÚTemporaryDirectoryr   r   r   r   Ú
read_bytesÚrunÚoutputÚblocksÚ	Exceptionr   ÚwarningÚnameÚ_ground_footnotes)ÚlocatorÚfn_agentr,   ÚpageÚraw_imgrI   ÚlocatedÚtÚtablesÚ	footnotesÚpage_wÚpage_hÚwordsÚtmpÚgriddedÚimageÚresultrZ   Úexcs                      r8   Ú_page_regionsro   \   sL  é € õ *à—N‘N 7°Ó6×6€GÙ6=Ó>²g°�y‰y˜!Ÿ'™' 1§8¡8Ó,±g€FÐ>ØKM€IÞÜ&-×&7Ò&7¼
ÀCÓ&N× NÑˆ˜Ü×(Ò(°Ò=ÀÜ˜3“i -Ñ/ˆGÜ×#Ò#¤L°'ÄLÔR^Ó_×_Ð_Ù! w×'9Ñ'9Ó';ÈÑTˆEðØ'Ÿ|™|Ð-EÀuÐ,MÓN×N�ØŸ™×-Ñ-�÷ >ô & f¨g¸fÓMˆ	Ø˜Ð"Ð"ò! 7ùÚ>ñ !Oñ `ñ Oøäó Ü—’ÐIÈ4×QYÑQYÐ[^Ô_Ø•ûðú÷ >Õ=üsŒ   ‚FžD4ŸF§,D7Á)FÁ<D<Á=FÂ7E=ÃD>ÃE=Ã*EÄ E ÄEÄFÄ7FÄ>E=Å EÅ
E:Å$E5Å0E=Å5E:Å:E=Å=
FÆFc                óÌ  • [         S:X  aü  [        U 5       VVs/ s HW  u  pV[        US-   SR                  UR                  5      UR
                  UR                  UR                  UR                  S9PMY     nnn[        XrX4[        [        5      n[        XƒU[        5      n[        [        U5      [        U5      -   X#U5      n	U	[!        U5      S n
[#        X
5       VVs/ s H  u  pkUR                  UR$                  4PM     snn$ / nU  HI  n[&        R(                  " XÒX4[        [        5      u  pïnUc  M,  UR+                  UR                  U45        MK     U$ s  snnf s  snnf )a|  Ground footnote-block flags to precise regions.

"prod": feed the flags through the production geometry chain the tables use
- flags_to_located (which runs tighten_region against the text layer),
pad_boxes, then declash_stacked over the combined table+footnote set so a
footnote box is pulled off the table directly above it. "marker": the probe's
bespoke marker/prose heuristic.
r   r   Ú )rP   rQ   Ú	row_startÚrow_endÚ	col_startÚcol_endN)Ú	GROUNDINGÚ	enumerater   ÚjoinÚmarkersrr   rs   rt   ru   r   r   r   r   r
   r   ÚlistÚlenÚziprR   ÚffpÚ_ground_blockr*   )rZ   Úlocated_tablesri   rg   rh   r5   ÚbÚflagsÚ
fn_locatedÚcombinedÚgroundedÚgr   ÚblkÚboxÚ_textÚ_markers                    r8   r^   r^   ~   sD  € ô �FÓô " &Ô)ô

ò *‘�ô Ø˜A™Ø—h‘h˜qŸy™yÓ)ØŸ+™+ØŸ	™	ØŸ+™+ØŸ	™	ôñ *ð 	ñ 

ô & e°FÄLÔR^Ó_ˆ
Ü˜z°6¼;ÓGˆ
Ü"¤4¨Ó#7¼$¸zÓ:JÑ#JÈEÐ[aÓbˆØœC Ó/Ð0Ð1ˆÜ25°fÔ2GÔHÒ2G©$¨!�—‘˜AŸH™HÓ%Ñ2GÒHÐHàEG€CÛˆÜ!×/Ò/°¸FÌLÔZfÓgÑˆ�GØ‹?Ø�J‰J˜Ÿ™ SÐ)Ö*ñ ð €Jùó-

ùó Is   ™AEÃ$E c           	     óŒ  • XS-
     nUR                   R                  UR                   R                  peU H€  u  pxn	[        X•U5      n
UR	                  U
[
        SS9  U(       a  SU SU SU 3OSU SU 3nUR                  U
R                  S-   [        SU
R                  S	-
  5      4US S
 S[
        S9  M‚     U H†  u  pÉ[        X•U5      n
UR	                  U
[        SSS9  U(       a  SSR                  U5      -   OSnUR                  U
R                  S-   [        US-
  U
R                  S-   5      4US S S[        S9  Mˆ     g )Nr   gffffffö?)ÚcolorÚwidthÚTÚ.z: r   g       @é   éP   é   )Úfontsizer‹   z[3 2] 0)r‹   rŒ   Údasheszfn rq   Úfné   r   )ÚrectrŒ   ÚheightÚ_rectÚ	draw_rectÚ	TABLE_RGBÚinsert_textÚx0ÚmaxÚy0ÚFOOTNOTE_RGBrx   r$   Úy1)r0   Úpage_nore   rf   ra   ÚwÚhrP   rQ   rR   r–   Úlabelry   s                r8   Ú_drawr¥   §   s>  € ð ˜‰{Ñ€DØ�9‰9�?‰?˜DŸI™I×,Ñ,€qÛ"(Ñˆ˜Ü�V Ó"ˆØ�‰�t¤9°CˆÑ8Þ49�!�G�9˜A˜g˜Y b¨¨Ñ0ÀÀ7À)È1ÈWÈIÐ?VˆØ×Ñ˜$Ÿ'™' A™+¤s¨3°·±¸!±Ó'<Ð=¸uÀSÀb¸zÐTUÔ]fÐÓgñ	 #)ó
 %‰ˆÜ�V Ó"ˆØ�‰�t¤<°sÀ9ˆÑMÞ-4�˜Ÿ™ Ó)Ò)¸$ˆØ×Ñ˜$Ÿ'™' A™+¤s¨1¨q©5°$·'±'¸A±+Ó'>Ð?ÀÀsÈÀÐVWÔ_kÐÓlò	 %r:   c           	     óž   • U u  p4pV[         R                  " [        X55      U-  [        XF5      U-  [        X55      U-  [        XF5      U-  5      $ r>   )r!   ÚRectr$   r�   )rR   r¢   r£   rœ   rž   Úx1r    s          r8   r˜   r˜   »   sA   € Ø�N€BˆBÜ�9Š9”S˜“[ 1‘_¤c¨"£k°A¡o´s¸2³{ÀQ±ÌÈBËÐVWÉÓXÐXr:   c           	   ƒ  óö  ^ ^^^^^#   • [         R                  " STR                  5        [        R                  " [        T5      5      n[        UR                  [        5      n[        R                  " SS9 n[        R                  " [        T[        U5      [        [        5      I S h  v•N m[        R                   " ["        5      mSUUU UU4S jjm[        R$                  " U4S j['        SUS-   5       5       SS06I S h  v•N nS S S 5        S	=px/ n	W Hƒ  n
[)        U
[*        5      (       a$  [         R,                  " S
TR                  U
5        M<  U
u  p¼n[/        X;XÍ5        U[1        U5      -  nU[1        U5      -  nU(       d  Mr  U	R3                  U5        M…     [4        R7                  SSS9  [4        TR8                  R                   STR:                   S3-  nUR=                  [        U5      5        UR?                  5         TR                  [        URA                  [B        5      5      UUUU	S.n[         R                  " STR                  UUUU	5        U$  GN· GN_! , (       d  f       GNd= f7f)Nzannotating {}zque270-pages-rJ   c           
   “  ó¾   >#   • T IS h  v•N   [        TTTU TU S-
     5      I S h  v•N sS S S 5      IS h  v•N   $  N2 N N	! , IS h  v•N  (       d  f       g = f7f)Nr   )ro   )ra   r`   Úimagesr_   r,   Úsems    €€€€€r8   ÚoneÚannotate_document.<locals>.oneÈ   sA   øé € ß’sÜ*¨7°H¸cÀ4ÈÐPTÐWXÑPXÑIYÓZ×Z÷ —s“sÙZ÷ —s—s�süsN   ƒAŠ=‹AŽA§?¨A«A·A¸A¿AÁAÁAÁ	AÁ
AÁAc              3  ó4   >#   • U  H  nT" U5      v •  M     g 7fr>   r?   )r@   rA   r­   s     €r8   rB   Ú$annotate_document.<locals>.<genexpr>Ì   s   øé € Ð(OÒ9N°A©¨Q¯¨Ò9Nùs   ƒr   Úreturn_exceptionsTr   zpage failed in {}: {}©ÚparentsÚexist_okÚ__z.annotated.pdf)r0   r   Úpagesre   Úfootnote_blocksÚfootnote_pagesz7{}: {} pages, {} tables, {} footnote blocks on pages {})ra   ÚintÚreturnzTuple[int, list, list])"r   Úinfor]   r!   r"   r#   r$   r%   ÚPAGE_CAPrU   rV   rS   rT   r9   r   ÚDPIÚ	SemaphoreÚPAGE_CONCURRENCYÚgatherr'   Ú
isinstanceÚBaseExceptionÚerrorr¥   r{   r*   ÚOUT_DIRÚmkdirÚparentÚstemr)   r+   Úrelative_toÚREPO)r_   r`   r,   r0   Ún_pagesrj   ÚresultsÚn_tablesÚn_fnÚfn_pagesÚresra   re   rf   Úout_pdfÚsummaryr«   r­   r¬   s   ```             @@@r8   Úannotate_documentrÒ   À   sð  ýé € Ü
‡K‚K� §¡Ô*Ü
�)Š)”C˜“HÓ
€CÜ�#—.‘.¤(Ó+€GÜ	×	$Ò	$¨OÒ	<ÀÜ×(Ò(¬¸¼dÀ3»iÌÌhÓW×WˆÜ×ÒÔ 0Ó1ˆ÷	[ó 	[ô  ŸšÔ(O¼¸qÀ'ÈAÁ+Ô9NÓ(OÐhÐcgÑh×hˆ÷ 
=ð Ð€HØ€HÛˆÜ�cœ=×)Ñ)Ü�LŠLÐ0°#·(±(¸CÔ@ÙØ"%Ñˆ�iÜˆc˜Ô+Ø”C˜“KÑˆØ”�I“Ñˆßˆ9Ø�O‰O˜DÖ!ñ ô ‡M�M˜$¨€MÑ.Ü˜3Ÿ:™:Ÿ?™?Ð+¨2¨c¯h©h¨Z°~ÐFÑF€GØ‡H�HŒS�‹\ÔØ‡I�I„Kà�x‰xÜ�7×&Ñ&¤tÓ,Ó-ØØØØ"ñ€Gô ‡K‚KØAØ�‰ØØØØôð €NòU Xò i÷ 
=Ö	<üsK   ˆA/I9Á73I'Â*I!Â+AI'ÄI$ÄI'Ä
A=I9ÆCI9É!I'É$I'É'
I6É1I9c                óZ   • U R                  S5      (       a  [        U 5      $ [        U 5      $ )zGLocal path passes through; an s3:// URI materializes through the cache.zs3://)Ú
startswithr   r   )Útargets    r8   Ú_resolverÖ   ò   s(   € à×Ñ˜×!Ñ!Ü Ó'Ð'Ü�‹<Ðr:   c              ƒ  ó   #   • [        5         [        5       n[        R                  " [        R                  " [
        [        5      [        R                  5      n/ n[        U S5       H•  u  pE[        R                  " SU[        U 5      U5         [        U5      nUR                  [        XU5      I S h  v•N 5        [$        R'                  SSS9  [$        S-  R)                  [*        R,                  " USS	95        M—     [/        [*        R,                  " USS	95        [/        S
[$         35        g  N|! [         a>  n[        R                   " SXW5        UR                  U[#        U5      S.5         S nAN¹S nAff = f7f)Nr   z=== doc {}/{}: {} ===zdocument failed {}: {})r0   rÃ   Tr²   z
index.jsonr   )Úindentz#
Annotated PDFs + index.json under )r   r   r}   Ú_build_agentÚfootnote_only_promptr   r   ÚFootnoteFinderResultrw   r   r»   r{   rÖ   r*   rÒ   r[   rÃ   r#   rÄ   rÅ   Ú
write_textÚjsonÚdumpsÚprint)Útargetsr_   r`   Úindexr5   rÕ   r,   rn   s           r8   Úmainrâ   ù   s"  é € Ü„OÜ#Ó%€GÜ×ÒÜ× Ò ¤¬|Ó<¼c×>VÑ>Vó€Hð €EÜ˜w¨Ö*‰	ˆÜ�ŠÐ+¨Q´°G³¸fÔEð	=Ü˜6Ó"ˆCØ�L‰LÔ0°ÀCÓH×HÔIô 	�‰˜d¨TˆÑ2Ü	�<Ñ	×+Ñ+¬D¯JªJ°uÀQÑ,GÖHñ +ô 
Œ$�*Š*�U 1Ñ
%Ô&Ü	Ð0´°	Ð
:Õ;ñ IøÜó 	=Ü�LŠLÐ1°6Ô?Ø�L‰L ´#°c³(Ñ;×<Ñ<ûð	=üsC   ‚BE>Â%D3Â4D1
Â5D3Â=A4E>Ä1D3Ä3
E;Ä=4E6Å1E>Å6E;Å;E>Ú__main__r   z--groundingÚ
annotated_)
r,   r   r-   r   r.   r¹   r/   r¹   rº   ú
List[Path])rº   rå   )r_   r   r`   Úobjectr,   r   ra   r¹   rb   r   rº   zTuple[int, List[Tuple[int, str, Tuple[float, float, float, float]]], List[Tuple[List[str], Tuple[float, float, float, float]]]])rZ   rz   r   zList[LocatedTable]ri   ræ   rg   Úfloatrh   rç   rº   ú9List[Tuple[List[str], Tuple[float, float, float, float]]])
r0   zfitz.Documentr¡   r¹   re   z8List[Tuple[int, str, Tuple[float, float, float, float]]]rf   rè   rº   ÚNone)rR   z!Tuple[float, float, float, float]r¢   rç   r£   rç   rº   z	fitz.Rect)r_   r   r`   ræ   r,   r   rº   Údict)rÕ   r#   rº   zOptional[Path])rà   z	List[str]rº   ré   )AÚ__doc__Ú
__future__r   rS   rÝ   ÚsysrU   Úpathlibr   Útypingr   r   r   r!   Úlogurur   r7   Úinsertr#   Ú__file__ÚresolverÆ   Úfootnote_finder_prober}   Úquber.agents.completenessr	   Úquber.agents.grid_locatorr
   r   r   r   r   r   r   r   r   r   Úquber.files.cacher   r   r³   rÉ   rE   r½   r¿   r¼   rv   rÄ   r9   rš   rŸ   rG   ro   r^   r¥   r˜   rÒ   rÖ   râ   Ú__name__Úargvrá   Úgirà   rX   )rA   s   0r8   Ú<module>rû      s`  ðñõ. #ã Û Û 
Û Ý ß (Ñ (ã Ý à ‡�‡��‘3‘t˜H“~×-Ñ-Ó/×6Ñ6Ó7Ô 8Û #å 0÷÷ ÷ ÷ >áˆHƒ~×ÑÓ×'Ñ'¨Ñ*€ØÐ%Ñ%€Ø	€ØÐ Ø€ð
 €	Ù
ˆx‹.×
 Ñ
 Ó
"×
)Ñ
)¨EÑ
1°KÑ
?€ôð  €	Ø €ô2ð
#Ø"ð#àð#ð 
ð#ð ð	#ð
 ð#ð Eô#ðD&Øð&à&ð&ð ð&ð ð	&ð
 ð&ð ?ô&ðRmØ	ðmàðmð Eðmð Ið	mð
 
ômô(Yô
/ôdô<ð* ˆzÓØ�8‰8�A�Bˆ<€DØ˜ÓØ�Z‰Z˜Ó&ˆØ˜˜a™‘Lˆ	Ø��b˜1‘f�ÐÙ�8‹n×$Ñ$Ó&×-Ñ-°Ñ5¸*ÀYÀKÐ8PÑP€GÞ‰d±´Ó :²¨A¡ Q¦±Ñ :€GØ‡K‚K‘�W“Õð ùò !;s   Æ=G,