Coverage for src / quber / core / extractors / base.py: 98%
141 statements
« prev ^ index » next coverage.py v7.14.0, created at 2026-09-23 22:14 -0400
1"""
2TableExtractor Protocol and ExtractedTable Pydantic model.
4`ExtractedTable` carries the four LLM-correction outputs (title, subtitle,
5markdown, footnotes) plus the Camelot-side provenance (page, bbox,
6flavor). The numeric content lives inside `markdown` — Camelot is the
7source of truth; the LLM is forbidden from changing values (enforced by
8the system prompt in `quber.agents.llm_client`).
9"""
11from __future__ import annotations
13import hashlib
14import json
15from pathlib import Path
16from typing import List, Literal, Optional, Protocol, Sequence, Tuple, runtime_checkable
18from pydantic import BaseModel, Field, field_validator
20from quber.agents.classifier import ClassifierResult
21from quber.agents.llm_client import FootnoteDef
24@runtime_checkable
25class TableExtractor(Protocol):
26 async def extract_tables(self, source: Path) -> List[ExtractedTable]: ...
28 def extract_tables_sync(self, source: Path) -> List[ExtractedTable]: ...
31class ExtractedTable(BaseModel):
32 table_id: str = Field(
33 default="",
34 description=(
35 "Deterministic address of this table: '<doc>-p<page>-t<n>', where <doc> is the "
36 "source file stem, <page> the 1-indexed page, and <n> the locator's reading-order "
37 "ordinal on that page. A sub-table produced by splitting a fused region appends "
38 "'-s<k>'. The same document yields the same IDs on every run, so external "
39 "references (a review comment, a validation report, a UI link) stay valid across "
40 "runs. Empty for engines that do not set it."
41 ),
42 )
43 content_fingerprint: str = Field(
44 default="",
45 description=(
46 "First 8 hex characters of sha256 over Camelot's raw value grid. The same "
47 "document yields the same fingerprint on every run; a changed value at the same "
48 "table_id shows as a different fingerprint. Empty when no grid backed the table."
49 ),
50 )
52 # LLM-correction outputs
53 title: str = Field(
54 default="", description="The table's printed name, copied off the page; empty when none is printed"
55 )
56 caption: str = Field(
57 default="", description="The sentence that introduces the table, copied off the page; empty when none"
58 )
59 # No longer written. Kept so artifacts from earlier runs still load.
60 subtitle: str = Field(default="", description="Retained for older artifacts; new runs leave it empty")
61 markdown: str = Field(description="Corrected markdown table; numeric values from Camelot")
62 footnotes: List[FootnoteDef] = Field(
63 default_factory=list,
64 description=(
65 "Footnotes printed below the table, each a pair of the footnote's own marker "
66 "(empty for an unmarked general note) and its text. Read off the table image "
67 "by the correction agent; the image's formatting decides what is a marker."
68 ),
69 )
70 footnote_refs: List[str] = Field(
71 default_factory=list,
72 description=(
73 "Footnote reference markers carried on the table's headers or cells, read off the "
74 "table image — a superscript or parenthetical number, letter, or symbol that points "
75 "to a footnote: e.g. '1', '(1)', a letter 'a', an asterisk '*', a dagger '†' or "
76 "double-dagger '‡', or a section sign '§'. Written as it appears. Empty if the table "
77 "carries none. Survives superscripts the text layer drops; the demand-driven footnote "
78 "lookup keys off this."
79 ),
80 )
81 footnote_marks: List["LocatedMarker"] = Field(
82 default_factory=list,
83 description=(
84 "Where each footnote reference marker sits: one entry per carrying cell, "
85 "addressed by row/col in the corrected markdown (header row 0). The agent "
86 "QUOTES the carrying cell and a lookup resolves the quote to its cell — "
87 "it never counts positions. An entry with row/col None is a marker whose "
88 "quote matched no cell (or that was catalogued without a quote) — kept "
89 "and flagged, never dropped."
90 ),
91 )
92 units: str = Field(default="", description="Scale/currency unit attribution caption; empty if none")
93 header_rows: Optional[int] = Field(
94 default=None,
95 description=(
96 "Leading column-header rows of the corrected markdown, counted off the table "
97 "image by the correction review. A stacked header counts every stacked row; 0 "
98 "means the fragment starts at data. None on artifacts extracted before this "
99 "capture existed."
100 ),
101 )
103 # Camelot-side provenance
104 page: int = Field(description="1-indexed page number")
105 bbox: Optional[Tuple[float, float, float, float]] = Field(
106 default=None, description="Camelot's geometric box on the page (x1, y1, x2, y2)"
107 )
108 som_region: Optional[Tuple[float, float, float, float]] = Field(
109 default=None,
110 description=(
111 "Set-of-Mark vision boundary box that located this table, normalized 0..1 with the page "
112 "top-left as origin (x1, y1, x2, y2). Full-precision floats, never floored or snapped to a "
113 "grid cell. None for engines that do not run the Set-of-Mark locator."
114 ),
115 )
116 content_region: Optional[Tuple[float, float, float, float]] = Field(
117 default=None,
118 description=(
119 "The table's actual content extent, normalized 0..1 top-left origin (x1, y1, x2, y2). "
120 "Same box as som_region but with the bottom revised down to the last tabular row, located "
121 "by matching the corrected markdown's last row back to the page text layer. som_region and "
122 "bbox can overrun past the grid (e.g. enclosing footnote lines printed below it); this is "
123 "the authoritative end of the table. Equal to som_region when no overrun is found or the "
124 "last row could not be located. None for engines without the Set-of-Mark locator."
125 ),
126 )
127 cell_grid: List[List["GroundedCell"]] = Field(
128 default_factory=list,
129 description=(
130 "The Camelot cell grid with per-cell geometry, as extracted before LLM "
131 "structure correction. Rows and columns align with Camelot's raw grid, "
132 "not the (possibly restructured) `markdown` — values are preserved verbatim "
133 "through correction, so the geometry can be remapped onto the corrected "
134 "cells by value later. Empty when no Camelot grid backed the table."
135 ),
136 )
137 corrected_grid: List[List["GroundedCell"]] = Field(
138 default_factory=list,
139 description=(
140 "The corrected markdown's grid with per-cell geometry — one GroundedCell per "
141 "markdown cell, same row/col addressing (header row 0), boxes normalized 0..1 "
142 "top-left. This is the complete cell-level view, fully resolved at the grounding "
143 "stage: combined cells carry the box their merge's reported addresses resolved to, "
144 "and every other cell its Camelot cell box via order-preserving alignment to the "
145 "raw grid — whole-cell matches first, then fragments of glued Camelot cells by "
146 "ordered span tiling within the aligned rows. box is None only where a cell's text "
147 "has no home in the raw grid; those gaps are warned in the run log at extraction. "
148 "Equal to cell_grid when no correction was applied."
149 ),
150 )
151 merged_cells: List["MergedCellBox"] = Field(
152 default_factory=list,
153 description=(
154 "Corrected cells that combine two or more source cells (a rejoined split "
155 "symbol, or a flattened multi-row/spanning header), each addressed by its "
156 "row/col in the corrected markdown and resolved to one Camelot cell box per "
157 "reported source address plus their union. Empty when no cells were combined."
158 ),
159 )
160 dropped_text: List[str] = Field(
161 default_factory=list,
162 description=(
163 "Text printed in the table's header area that appears in no output cell — "
164 "usually surrounding page text the correction rightly excluded, but recorded "
165 "so an omission is never silent. Each entry becomes a table-level review "
166 "flag (status 'header_text_dropped'). Empty when everything printed was carried."
167 ),
168 )
169 flavor: Literal["lattice", "stream", "unknown"] = Field(
170 default="unknown", description="Camelot extraction flavor that produced this table"
171 )
172 kind: Literal["text_table", "chart", "image_table"] = Field(
173 default="text_table",
174 description=(
175 "What this region really is, set by reconciliation against docling. "
176 "text_table: a real text-layer table (the default). chart: SoM mistook "
177 "a docling-classified picture (bar/line chart, etc.) for a table. "
178 "image_table: a real table rendered as an image (no text layer under a "
179 "docling-detected table, Camelot empty). Its content is whatever last "
180 "read the image: the page-scan workflow's reading where that has run, "
181 "and docling's own OCR of it where it has not."
182 ),
183 )
184 source: Optional[str] = Field(default=None, description="Source PDF path")
186 # What each pipeline stage did to this table: parse quality, classifier verdict, LLM edits
187 camelot_accuracy: float = Field(
188 default=0.0,
189 ge=0.0,
190 le=100.0,
191 description="Camelot parsing_report.accuracy (0-100); how cleanly text snapped to detected cells",
192 )
193 classifier_decision: Optional[ClassifierResult] = Field(
194 default=None,
195 description="What the TableClassifier decided about this candidate; None if classifier was bypassed",
196 )
197 llm_corrected: bool = Field(
198 default=False,
199 description="True if the LLM modified Camelot's raw markdown during the correction pass",
200 )
202 @field_validator("footnotes", mode="before")
203 @classmethod
204 def coerce_legacy_footnote_strings(cls, v: object) -> object:
205 """Artifacts extracted before footnotes carried their marker stored each
206 footnote as a bare string. Load those as unmarked notes — the text is
207 intact; only the marker pairing is absent — so old artifacts stay
208 readable."""
209 if isinstance(v, list):
210 return [{"marker": "", "text": item} if isinstance(item, str) else item for item in v]
211 return v
213 # Audit trail of the detect/match/audit flow; None when the legacy classifier path produced the table
214 extraction_record: Optional[ExtractionRecord] = Field(
215 default=None,
216 description="Record of the detector/correspondence/completeness flow; None if not used",
217 )
220class ExtractionRecord(BaseModel):
221 """Outcome of the extraction attempt for one detected table.
223 Produced by `CamelotCorrespondenceExtractor`. Its pipeline has a vision
224 detector list every table visible on a page, matches each detected
225 table to the Camelot chunk(s) holding its data, and audits the result
226 for completeness against the page image.
228 This record captures the status of that attempt, the chunks that supplied
229 the data, and the audit verdict — including the failure case where a
230 table was detected by the vision detector but not extracted by Camelot
231 (`detected_not_extracted`); so a table the detector saw is never silently
232 dropped from the output.
234 None on `ExtractedTable` when the table came through the legacy
235 classifier/unifier path, which has no detector to anchor a record to.
236 """
238 status: Literal["extracted", "incomplete", "detected_not_extracted"] = Field(
239 description=(
240 "extracted: a Camelot chunk matched the detected table and passed the "
241 "completeness audit. incomplete: matched but the completeness audit "
242 "found numbers in the image absent from the data, and no chunk could "
243 "supply them. detected_not_extracted: the detector confirmed a table on "
244 "the page but neither Camelot flavor nor the recovery pass produced it "
245 "(reported explicitly, never silently dropped)."
246 ),
247 )
248 detected_ordinal: Optional[int] = Field(
249 default=None,
250 description="1-based top-to-bottom position of the matched detected table on its page",
251 )
252 detected_description: str = Field(
253 default="",
254 description="The detector's description of this table; the content hook correspondence matched on",
255 )
256 matched_flavor: Optional[Literal["lattice", "stream"]] = Field(
257 default=None,
258 description="Which Camelot flavor supplied the chunk(s); None for detected_not_extracted",
259 )
260 source_candidate_ids: List[str] = Field(
261 default_factory=list,
262 description="Camelot candidate_ids assembled into this table, in page order",
263 )
264 combined: bool = Field(
265 default=False,
266 description=(
267 "True if a single Camelot chunk spans two or more detected tables "
268 "(convergence). The chunk is kept whole and flagged, never split."
269 ),
270 )
271 combined_ordinals: List[int] = Field(
272 default_factory=list,
273 description="Detected ordinals a combined chunk spans; empty unless combined is True",
274 )
275 completeness_complete: Optional[bool] = Field(
276 default=None,
277 description="completeness audit verdict; None if the audit did not run (e.g. detected_not_extracted)",
278 )
279 completeness_gap: str = Field(
280 default="",
281 description="If truncated, which edge was cut off (per the completeness audit); empty when complete",
282 )
283 filled_from_text_layer: bool = Field(
284 default=False,
285 description=(
286 "True if the text-layer fill step inserted one or more figures to round off an "
287 "edge truncation. The inserted values are listed in filled_cells; every other cell "
288 "remains Camelot-sourced."
289 ),
290 )
291 filled_cells: List[FilledCell] = Field(
292 default_factory=list,
293 description="Figures inserted by the fill step, with their grid position and text-layer origin; empty unless filled_from_text_layer",
294 )
297class FilledCell(BaseModel):
298 """One figure inserted into a table by the text-layer fill step.
300 Records that this value did NOT come from a Camelot chunk: it was read
301 directly from the PDF text layer at (x, y) and placed into the grid to
302 round off an edge truncation. Camelot remains the source for every
303 other cell; this keeps the amendment auditable and never blurs a
304 filled value with a Camelot-extracted one (the source field is always
305 `text_layer`).
306 """
308 value: str = Field(description="The literal text-layer figure that was inserted")
309 column: int = Field(
310 description="0-based column index in the assembled row the value was placed in (0 is the label column)"
311 )
312 row_label: str = Field(default="", description="Label of the restored row the value was placed into")
313 edge: Literal["top", "bottom"] = Field(description="Which edge the truncated row was restored at")
314 x: float = Field(description="Text-layer x of the figure (PDF points)")
315 y: float = Field(description="Text-layer y of the figure (PDF points)")
316 source: Literal["text_layer"] = Field(
317 default="text_layer",
318 description="Always text_layer; distinguishes a filled cell from a Camelot-sourced one",
319 )
322class LocatedMarker(BaseModel):
323 """One footnote reference marker tied to the cell that carries it.
325 `row`/`col` address the carrying cell in the corrected markdown (header
326 row 0), resolved by looking up the cell text the agent QUOTED for the
327 marker — the model never counts positions. Both None when the marker is
328 known to be on the table but its carrying cell is not — the marker is
329 kept and flagged for review rather than guessed onto a row.
330 """
332 marker: str = Field(description="The reference marker as printed: '1', '(1)', 'a', '*', a dagger")
333 row: Optional[int] = Field(
334 default=None,
335 ge=0,
336 description="0-based row of the carrying cell in the corrected markdown; None when unplaced",
337 )
338 col: Optional[int] = Field(
339 default=None,
340 ge=0,
341 description="0-based column of the carrying cell in the corrected markdown; None when unplaced",
342 )
343 kind: Literal["footnote", "section"] = Field(
344 default="footnote",
345 description=(
346 "The correction agent's judgement of what the marker points at, made from "
347 "the page image: 'footnote' — a note printed for this table; 'section' — a "
348 "cross-reference to a named part of the document whose content lives "
349 "elsewhere. Resolution routes on this: section references resolve to a "
350 "heading pointer and are never hunted as footnotes."
351 ),
352 )
353 scope: Literal["cell", "table"] = Field(
354 default="cell",
355 description=(
356 "'cell': the marker is carried by a grid cell at row/col. 'table': the "
357 "marker is carried by the table's own title or subtitle text — a spanning "
358 "band or title suffix — so it qualifies the whole table and reaches every "
359 "record; row/col are None but the marker is placed, not flagged."
360 ),
361 )
364class CellStatus(BaseModel):
365 """One entry of the cell-status reference: the machine code a corrected
366 cell carries and the human framing a front end shows for it. `inspect`
367 marks the codes a consumer surfaces to the user when the table is used.
368 `cases` names the residue-catalog cases the code covers (the catalog is
369 docs/CELL_STATUS_CASES.pdf), so a status always points back to a
370 demonstrated example.
371 """
373 code: str
374 label: str
375 description: str
376 inspect: bool = False
377 cases: str = ""
380# The canonical status registry. The classifier may only emit these codes —
381# GroundedCell validates against it — so a front end can key a lookup table
382# on `code` without ever meeting an unknown value. Full prose glossary:
383# set_of_mark/README.md.
384CELL_STATUS_REFERENCE: Tuple[CellStatus, ...] = (
385 CellStatus(
386 code="reconciled",
387 label="Matches the page.",
388 description=(
389 "The cell's text tied back to a printed source and carries that source's measured coordinates."
390 ),
391 ),
392 CellStatus(
393 code="header_printed_unlocated",
394 label="This header is printed on the page. We could not measure where.",
395 description=(
396 "Header or band text is printed on the page, but its coordinates "
397 "could not be measured. The text is correct; only the geometry is "
398 "missing. Expected behavior — a pass, not a defect."
399 ),
400 cases="1",
401 ),
402 CellStatus(
403 code="label_printed_unlocated",
404 label="This row label is printed on the page. We could not measure where.",
405 description=(
406 "A row label is printed on the page, usually wrapped across printed "
407 "lines, but its coordinates could not be measured. Expected "
408 "behavior — a pass, not a defect."
409 ),
410 cases="2 and 3",
411 ),
412 CellStatus(
413 code="single_character",
414 label='A one-character cell, such as a bare "$". Not matched to a position by design.',
415 description=(
416 "One-character text (a bare '$'), excluded from location matching by "
417 "design: a bare symbol or digit would anchor inside any unrelated "
418 "number. Working as designed — a pass, not a defect."
419 ),
420 cases="7",
421 ),
422 CellStatus(
423 code="total_label_added",
424 label='The page prints no label on the totals row, so we wrote "Total".',
425 description=(
426 "The totals row is printed with no label; the extraction wrote "
427 "'Total'. Flagged for user review because the author could have "
428 "intended something else — a review item, NOT a defect."
429 ),
430 inspect=True,
431 cases="5",
432 ),
433 CellStatus(
434 code="header_label_added",
435 label="The page prints no header over this column, so we named it.",
436 description=(
437 "The table is printed with no header over this column; the "
438 "extraction wrote a generic column name ('Item', 'Description'). "
439 "Flagged for user review because the author could have intended "
440 "something else — a review item, NOT a defect."
441 ),
442 inspect=True,
443 cases="6",
444 ),
445 CellStatus(
446 code="unverified",
447 label="This value is on the page, but our check of it did not complete. Worth a manual look.",
448 description=(
449 "The condition could not be confirmed: the text has no printed source "
450 "and is not an authorized conventional label, or the inspector could "
451 "not tell from the image. An open question for user review — not yet "
452 "a defect."
453 ),
454 inspect=True,
455 ),
456 CellStatus(
457 code="defect",
458 label="The page shows something different from what we recorded. See the note.",
459 description=(
460 "The inspector positively observed the page showing something OTHER "
461 "than what the extraction recorded — a different word printed at an "
462 "added label's position, or a structure the condition misdescribes. "
463 "A confirmed defect; the evidence names what the page shows."
464 ),
465 inspect=True,
466 ),
467 CellStatus(
468 code="footnote_unresolved",
469 label="This cell has a footnote marker, and we could not find the footnote it refers to.",
470 description=(
471 "A marker is printed on a table cell or a figure's label and no tier "
472 "found the note it points at — not the pairs the correction agent "
473 "read off the image, not the reading-order scan, not the demand-driven "
474 "lookup. The element states a figure the document qualifies somewhere, "
475 "and the qualification is missing: read alone it looks unqualified, "
476 "which is indistinguishable from a clean reading. An element-level "
477 "flag naming the marker, not an output cell."
478 ),
479 inspect=True,
480 ),
481 CellStatus(
482 code="footnote_unreferenced",
483 label="A footnote on this page that no marker points to.",
484 description=(
485 "A note printed for a table or figure that no marker on it names. "
486 "Currently judged within one element type, so a note a table claims "
487 "can still read as unreferenced by a figure on the same page and the "
488 "reverse — which is why it is catalogued but not surfaced. Becomes "
489 "meaningful once every reader's claims are pooled and the leftovers "
490 "reported once for the document."
491 ),
492 ),
493 CellStatus(
494 code="footnote_marker_unplaced",
495 label="This table has a footnote marker we could not tie to a cell.",
496 description=(
497 "The correction agent read this footnote reference marker off the "
498 "table image but could not tie it to a carrying cell, or the "
499 "position it reported failed validation. The marker is real; only "
500 "its cell is unknown, so its footnote can be attached at table "
501 "level but not to a specific row. A table-level flag: it names "
502 "the marker, not an output cell."
503 ),
504 inspect=True,
505 ),
506 CellStatus(
507 code="header_text_dropped",
508 label="The page prints header text here that we left out of the table.",
509 description=(
510 "Text printed in the table's header area appears in no output cell. "
511 "Usually surrounding page text (a heading, a note) that the "
512 "extraction correctly excluded from the table, but surfaced so an "
513 "omission is never silent. A table-level flag: it names the missing "
514 "text, not an output cell."
515 ),
516 inspect=True,
517 cases="4",
518 ),
519 CellStatus(
520 code="value_misread",
521 label="The page prints a different figure here. Check which is right.",
522 description=(
523 "A chart value where the page fragment at the value's position "
524 "states a different number than the reading. One of the two is "
525 "wrong and the position pins exactly where to look, so the value "
526 "never passes silently. A figure-value flag: it names the value "
527 "and carries the fragment's box."
528 ),
529 inspect=True,
530 ),
531 CellStatus(
532 code="value_unreconciled",
533 label="Read from the chart once, without a second check. Worth a manual look.",
534 description=(
535 "A chart value present in one reading with no counterpart in the "
536 "other. The note states the direction: a scan-only value the local "
537 "read did not corroborate, or a locally read value the scan "
538 "omitted. Either way the value stands unverified and is surfaced "
539 "rather than passed. A figure-value flag."
540 ),
541 inspect=True,
542 ),
543)
545# The conventional labels the correction prompt authorizes it to add where the
546# page prints nothing. Kept in lockstep with the UNLABELED TOTAL ROW / MISSING
547# HEADER ROW guidance in the vet-structure prompt: an added label outside this
548# set classifies `unverified`, never as an added-label status.
549AUTHORIZED_LABELS = frozenset({"total", "item", "description"})
551_CELL_STATUS_CODES = frozenset(s.code for s in CELL_STATUS_REFERENCE)
554class GroundedCell(BaseModel):
555 """One Camelot cell paired with its geometry.
557 `text` is the cell value (immutable through the pipeline). `box` is the
558 cell's box normalized 0..1 with the page top-left as origin — the same
559 frame as `som_region` and `MergedCellBox`, so every box the table carries
560 reads in ONE frame — or None when no Camelot box backs the cell.
561 """
563 text: str = Field(description="Cell value from Camelot's grid")
564 box: Optional[Tuple[float, float, float, float]] = Field(
565 default=None,
566 description="Cell box (x1, y1, x2, y2) normalized 0..1, top-left origin; None if unavailable",
567 )
568 status: Optional[str] = Field(
569 default=None,
570 description=(
571 "Why the cell does or does not carry a box; set on corrected-grid cells only. "
572 "One of the codes in CELL_STATUS_REFERENCE — see that registry (and the "
573 "set_of_mark README) for each code's meaning and whether a consumer should "
574 "surface it for user inspection."
575 ),
576 )
578 note: Optional[str] = Field(
579 default=None,
580 description=(
581 "Inspection evidence for a flagged cell: the status inspector's one-line "
582 "account of what the table image shows at this position. None when the "
583 "cell needed no inspection."
584 ),
585 )
587 @field_validator("status")
588 @classmethod
589 def status_is_a_registered_code(cls, v: Optional[str]) -> Optional[str]:
590 if v is not None and v not in _CELL_STATUS_CODES:
591 raise ValueError(f"unknown cell status {v!r}; register it in CELL_STATUS_REFERENCE")
592 return v
595class CellFlag(BaseModel):
596 """One item a run surfaces for review — the error-check record.
598 Carries full document identity (source, page, table) alongside the item,
599 so a flag can be traced to its table without reverse attribution from
600 page numbers. `status` is a registered inspect code; `note` is the
601 status inspector's one-line account of what the page image shows. Most
602 flags point at an output cell; a table-level flag (status `header_text_dropped`,
603 printed text that reached no output cell) has no cell to point at, so
604 `row` and `col` are None and `text` holds the missing text.
605 """
607 source: str
608 page: int
609 table_id: Optional[str] = None
610 title: str = ""
611 row: Optional[int] = None
612 col: Optional[int] = None
613 text: str
614 status: str
615 note: Optional[str] = None
618def cell_flags(tables: Sequence["ExtractedTable"]) -> List[CellFlag]:
619 """Every corrected cell whose status is registered for inspection, plus
620 each table's dropped printed text, with document identity attached — the
621 run's review queue, ready to emit."""
622 inspect_codes = frozenset(s.code for s in CELL_STATUS_REFERENCE if s.inspect)
623 out: List[CellFlag] = []
624 for t in tables:
625 for r, row in enumerate(t.corrected_grid or []):
626 for c, cell in enumerate(row):
627 if cell.status in inspect_codes:
628 out.append(
629 CellFlag(
630 source=t.source or "",
631 page=t.page,
632 table_id=t.table_id,
633 title=t.title or "",
634 row=r,
635 col=c,
636 text=cell.text,
637 status=cell.status or "",
638 note=cell.note,
639 )
640 )
641 for fragment in t.dropped_text:
642 out.append(
643 CellFlag(
644 source=t.source or "",
645 page=t.page,
646 table_id=t.table_id,
647 title=t.title or "",
648 text=fragment,
649 status="header_text_dropped",
650 )
651 )
652 for mark in t.footnote_marks:
653 if mark.row is None and mark.scope != "table":
654 out.append(
655 CellFlag(
656 source=t.source or "",
657 page=t.page,
658 table_id=t.table_id,
659 title=t.title or "",
660 text=mark.marker,
661 status="footnote_marker_unplaced",
662 )
663 )
664 return out
667class MergedCellBox(BaseModel):
668 """A corrected cell built by combining two or more source cells, resolved to
669 per-source-cell boxes plus their union.
671 The correction agent reads the Camelot grid inside a printed spreadsheet
672 coordinate frame and reports each merge's source cells by ADDRESS
673 (`source_cells`, e.g. ['B2', 'B3']); `source_boxes` holds one box per
674 address, in the same order — the named cell's Camelot-measured box, or None
675 where the address could not be resolved (bad label, blank cell, or text
676 that does not appear in the result). `box` is the union of the located
677 boxes, a coarse envelope over the whole combined cell. All boxes are
678 normalized 0..1 with the page top-left as origin — the same frame as
679 `som_region`. Every box is Camelot's own measurement; the model only ever
680 repeats printed labels.
681 """
683 result: str = Field(description="The combined cell text, as it appears in the corrected markdown")
684 row: int = Field(
685 ge=0,
686 description=(
687 "0-based row of this cell in the corrected markdown table (header row is 0) — the exact "
688 "address of the cell, since result text alone repeats (e.g. an identical total in the "
689 "Basic and Diluted EPS rows)"
690 ),
691 )
692 col: int = Field(ge=0, description="0-based column of this cell in the corrected markdown table")
693 sources: List[str] = Field(default_factory=list, description="The source cell values that were combined")
694 source_cells: List[str] = Field(
695 default_factory=list,
696 description=(
697 "The printed coordinate of each combined Camelot cell as the agent read it off the "
698 "coordinate frame (e.g. ['B2', 'B3']); resolves into cell_grid by lookup"
699 ),
700 )
701 source_boxes: List[Optional[Tuple[float, float, float, float]]] = Field(
702 default_factory=list,
703 description="One box per source_cells entry (same order), normalized 0..1 top-left; None where unresolved",
704 )
705 box: Optional[Tuple[float, float, float, float]] = Field(
706 default=None,
707 description="Union of the located source boxes (x1, y1, x2, y2) normalized 0..1, top-left origin; None if none located",
708 )
709 grounded_by: Literal["cell_address", "partial", "none"] = Field(
710 default="none",
711 description=(
712 "cell_address: every reported source address resolved and text-validated; partial: "
713 "some resolved (the union box under-covers the combined cell); none: no address "
714 "reported or none resolved"
715 ),
716 )
719def table_address(source: str | Path, page: int, ordinal: int) -> str:
720 """The deterministic table ID: '<doc>-p<page>-t<ordinal>'.
722 Built purely from the table's position, so the same document yields the
723 same IDs on every run. A split sub-table appends '-s<k>' to its parent's
724 address at the split site.
725 """
726 return f"{Path(source).stem}-p{page}-t{ordinal}"
729def grid_fingerprint(cells: List[List[str]]) -> str:
730 """First 8 hex characters of sha256 over the raw value grid."""
731 if not cells:
732 return ""
733 return hashlib.sha256(json.dumps(cells).encode()).hexdigest()[:8]
736def grounded_grid(
737 cells: List[List[str]],
738 cell_boxes: Optional[List[List[Optional[Tuple[float, float, float, float]]]]],
739 page_w: float,
740 page_h: float,
741) -> List[List[GroundedCell]]:
742 """Pair a Camelot cell grid with its per-cell geometry into GroundedCells.
744 `cell_boxes` arrive in Camelot's frame (PDF points, bottom-left origin) and
745 are converted here to normalized 0..1 top-left, so the grid serializes in
746 the same frame as every other box on the table. Shaped exactly like
747 `cells`. Any position `cell_boxes` does not cover (or a None `cell_boxes`)
748 yields a cell with no box, so the result is safe to build from a candidate
749 whose geometry was not captured.
750 """
752 def norm(box: Optional[Tuple[float, float, float, float]]) -> Optional[Tuple[float, float, float, float]]:
753 if box is None or page_w <= 0 or page_h <= 0:
754 return None
755 x1, y1, x2, y2 = box
756 return (x1 / page_w, (page_h - y2) / page_h, x2 / page_w, (page_h - y1) / page_h)
758 grid: List[List[GroundedCell]] = []
759 for i, row in enumerate(cells):
760 box_row = cell_boxes[i] if cell_boxes and i < len(cell_boxes) else []
761 grid.append(
762 [
763 GroundedCell(text=txt, box=norm(box_row[j] if j < len(box_row) else None))
764 for j, txt in enumerate(row)
765 ]
766 )
767 return grid