Coverage for src / quber / core / fusion / graft.py: 88%
327 statements
« prev ^ index » next coverage.py v7.14.0, created at 2026-09-23 22:14 -0400
1"""Build the unified document: move Camelot bodies into the docling spine.
3The matcher classified every region; this pass acts on those records with
4docling-core's native mutation API, so no custom serializer is needed:
6- `replace` — rebuild each matched docling table's body from the Camelot cells,
7 in place, and attach the SoM title/subtitle/units/footnotes. The spine element
8 and its reading-order position are untouched.
9- `docling_miss` — insert the Camelot table into the spine right after the
10 nearest element above it on the page (before the page's first element when the
11 table sits at the top), so reading order is preserved.
12- `docling_undercount` — delete the docling table(s) in the region and insert the
13 finer-grained Camelot tables in their place.
14- `image_table`, `chart` — no document change. docling already holds the OCR'd
15 table (image_table) or the picture (chart); only the SoM side is annotated.
16- `som_miss` — no change, surfaced as an error. A table docling found but SoM did
17 not is never synthesized from docling.
19The source document is never mutated: the caller clones it and this pass mutates
20the clone.
22Deleting and inserting tables renumbers every later table's `self_ref`, so after
23all mutation the pass rewrites each match's `docling_table_refs` to the grafted
24tables' final positions. The matches ship in the fusion report beside the
25unified document, and a consumer resolving a ref against that document must land
26on the table the match is about — a stale ref silently hands it a neighboring
27table's identity (its section heading, its position in the flow).
28"""
30from __future__ import annotations
32import re
33from collections import Counter
34from typing import Dict, List, Optional, Tuple
36from docling_core.types.doc.base import BoundingBox, CoordOrigin
37from docling_core.types.doc.document import (
38 DocItem,
39 DoclingDocument,
40 FloatingMeta,
41 NodeItem,
42 PictureMeta,
43 ProvenanceItem,
44 TableCell,
45 TableData,
46 TableItem,
47)
48from docling_core.types.doc.labels import DocItemLabel
49from loguru import logger
51from quber.core.extractors.base import ExtractedTable
52from quber.core.extractors.set_of_mark.merge_grounding import markdown_rows
53from quber.core.fusion.models import RegionMatch
54from quber.core.printed_text import printed_key
56#: Where an element records which readers have read its region, and what each
57#: produced. A later reader consults it rather than judging the region again.
58READ_BY_FIELD = "quber__read_by"
60#: The table engine that extracts from the PDF text layer.
61SET_OF_MARK = "set-of-mark"
63NormBox = Tuple[float, float, float, float]
64Box = Tuple[float, float, float, float]
67def build_unified_document(
68 document: DoclingDocument,
69 som_tables: List[ExtractedTable],
70 matches: List[RegionMatch],
71 page_dims: Dict[int, Tuple[float, float]],
72) -> Tuple[DoclingDocument, List[str]]:
73 """Graft the Camelot bodies into a clone of `document` per the matches.
75 Returns the unified document and a list of error strings (one per `som_miss`
76 region — a table docling found that SoM did not match). The input document is
77 not mutated.
78 """
79 unified = document.model_copy(deep=True)
80 tables_by_ref = {t.self_ref: t for t in unified.tables}
81 errors: List[str] = []
83 # Which TableItem(s) in the unified document realize each match. Deleting
84 # and inserting tables renumbers every later table's self_ref, so refs
85 # recorded against the pre-graft document go stale; the objects are
86 # tracked here and each match's refs are rewritten to their final
87 # positions once all mutation is done. Surviving tables map through
88 # `tables_by_ref` (the graft mutates them in place, never re-creates
89 # them); deleted-and-replaced regions map to the inserted tables.
90 realized: List[Tuple[RegionMatch, List[Tuple[str, Optional[TableItem]]]]] = []
92 for match in matches:
93 surviving = [(r, tables_by_ref.get(r)) for r in match.docling_table_refs]
94 if match.kind == "replace":
95 _apply_replace(unified, tables_by_ref, match, som_tables, page_dims)
96 realized.append((match, surviving))
97 elif match.kind == "docling_miss":
98 inserted, errs = _apply_insert(unified, match, som_tables, page_dims)
99 errors.extend(errs)
100 realized.append((match, [("", t) for t in inserted]))
101 elif match.kind == "docling_undercount":
102 inserted, errs = _apply_undercount(unified, tables_by_ref, match, som_tables, page_dims)
103 errors.extend(errs)
104 realized.append((match, [("", t) for t in inserted]))
105 elif match.kind == "som_merged":
106 # The split pass should have refined this region into a 1:1 replace
107 # before grafting. If a residual som_merged reaches here the split was
108 # declined or failed; docling's table count stands and its bodies are
109 # left untouched rather than grafting a fused body over split tables.
110 errors.append(
111 f"page {match.page}: som_merged not resolved by split "
112 f"({len(match.som_indices)} SoM vs {len(match.docling_table_refs)} docling); "
113 "docling tables left untouched"
114 )
115 realized.append((match, surviving))
116 elif match.kind == "som_miss":
117 errors.append(
118 f"page {match.page}: docling table {match.docling_table_refs} has no SoM match "
119 "(SoM/Camelot missed a table docling found)"
120 )
121 realized.append((match, surviving))
122 elif match.kind == "chart":
123 _mark_vetted(unified, match, som_tables)
124 realized.append((match, surviving))
125 else:
126 # image_table leaves the document unchanged, but its docling refs
127 # still shift when another region's graft deletes a table.
128 realized.append((match, surviving))
130 _dedup_table_region_text(unified, matches, som_tables, page_dims)
131 _rewrite_report_refs(unified, realized)
132 return unified, errors
135def _rewrite_report_refs(
136 unified: DoclingDocument,
137 realized: List[Tuple[RegionMatch, List[Tuple[str, Optional[TableItem]]]]],
138) -> None:
139 """Point every match's `docling_table_refs` at the unified document.
141 Final refs are read off each tracked object's position in the tables
142 array — the same positions serialization writes — so the rewrite cannot
143 disagree with the document on disk. A ref that never resolved to a table
144 is kept verbatim rather than dropped: it still names what the matcher saw,
145 and dropping it would hide the mismatch.
146 """
147 final_ref = {id(t): f"#/tables/{i}" for i, t in enumerate(unified.tables)}
148 for match, items in realized:
149 refs: List[str] = []
150 for original, item in items:
151 ref = final_ref.get(id(item)) if item is not None else None
152 if ref is None and not original:
153 continue # an inserted table later deleted; nothing to name
154 refs.append(ref if ref is not None else original)
155 match.docling_table_refs = refs
158#: docling labels that are loose body/attribution text, safe to drop when a fused
159#: table already represents them. Section headers, titles, and page headers/footers
160#: are deliberately excluded so the spine stays intact.
161_REDUNDANT_LABELS = {
162 DocItemLabel.TEXT,
163 DocItemLabel.PARAGRAPH,
164 DocItemLabel.CAPTION,
165 DocItemLabel.FOOTNOTE,
166}
169def _mark_vetted(document: DoclingDocument, match: RegionMatch, som_tables: List[ExtractedTable]) -> None:
170 """Record on a picture that the table engine has already read its region.
172 A `chart` match is a table the engine extracted from the text layer whose
173 region overlaps a picture the parse detected. That is routinely a real table
174 printed beside a chart, both inside the one region the engine bounded: the
175 engine read the table correctly and drew its box around the pair.
177 Nothing about the document changes. What is recorded is which reader has read
178 the region and what it produced, so a later reader leaves it alone instead of
179 stating the same table twice. The alternative is for that later step to
180 recompute the overlap against an artifact it is not handed, which is the same
181 judgement made again from less.
183 The tag names the reader rather than the outcome. A region already read is
184 the general condition; which tool did the reading is what a caller needs to
185 weigh it, and another tool added later says so the same way.
186 """
187 produced = [som_tables[i].table_id for i in match.som_indices if som_tables[i].table_id]
188 by_ref = {p.self_ref: p for p in document.pictures}
189 for ref in match.docling_picture_refs:
190 picture = by_ref.get(ref)
191 if picture is None:
192 continue
193 base = picture.meta or PictureMeta()
194 read_by = list(getattr(base, READ_BY_FIELD, None) or [])
195 read_by.append({"reader": SET_OF_MARK, "produced": produced})
196 picture.meta = base.model_copy(update={READ_BY_FIELD: read_by})
197 logger.info("Fusion: picture {} read by {} ({})", ref, SET_OF_MARK, ", ".join(produced))
200#: A docling item inside a table's region is a duplicate of the grafted table
201#: only when the table's own text accounts for its words. The two extractors
202#: render the same printed line with small differences (a curly against a
203#: straight apostrophe, a hyphen dropped at a line break), so a duplicate can
204#: miss a token or two; prose never comes close.
205_COVERAGE_MIN = 0.98
206_UNMATCHED_TOLERANCE = 1
208_TOKEN_RE = re.compile(r"[a-z0-9]+")
211def _dedup_table_region_text(
212 unified: DoclingDocument,
213 matches: List[RegionMatch],
214 som_tables: List[ExtractedTable],
215 page_dims: Dict[int, Tuple[float, float]],
216) -> None:
217 """Drop docling body text the grafted tables now represent.
219 docling emits a table's cell text — and its title/units/footnote lines — both
220 inside the TableItem and as loose TextItems in the reading-order flow, so the
221 unified output would show each fused table once as a clean table and again as
222 scattered fragments. For every region a SoM table covers, consider the docling
223 TextItems (text/paragraph/caption/footnote) whose center falls inside the SoM
224 region, and delete each one whose words the grafted table's own text accounts
225 for (its cells, title, subtitle, units, and footnotes); the grafted body plus
226 the structured caption/footnotes is then the single representation.
228 Position alone is never grounds for deletion. The locator's region is the
229 vision model's box, and it reaches past the table body: it is drawn to
230 enclose the footnote lines below the grid, and it can take in what is printed
231 under those too. Text inside the box that the table does not carry is not
232 table text; it is the box being too big. A paragraph stating what share of a
233 loan book floats, a sentence introducing the next table, a footnote whose
234 text continues on the next page and so exceeds what the table carries: each
235 stays in the document as the text it is. The separation is logged.
237 Section headers and page headers/footers are never removed, so the spine
238 survives. The attribution captions/footnotes added during the graft carry no
239 provenance box, so they are never matched here.
240 """
241 regions_by_page: Dict[int, List[Tuple[Box, ExtractedTable]]] = {}
242 for match in matches:
243 if match.kind not in ("replace", "docling_miss", "docling_undercount", "image_table"):
244 continue
245 for i in match.som_indices:
246 som = som_tables[i]
247 if som.som_region is None:
248 continue
249 width, height = page_dims.get(som.page, (612.0, 792.0))
250 x1, y1, x2, y2 = som.som_region
251 region = (min(x1, x2) * width, min(y1, y2) * height, max(x1, x2) * width, max(y1, y2) * height)
252 regions_by_page.setdefault(som.page, []).append((region, som))
254 if not regions_by_page:
255 return
257 bags: Dict[int, Counter[str]] = {}
258 victims: List[NodeItem] = []
259 for item in unified.texts:
260 if item.label not in _REDUNDANT_LABELS or not item.prov:
261 continue
262 page_no = item.prov[0].page_no
263 regions = regions_by_page.get(page_no)
264 if not regions:
265 continue
266 _, height = page_dims.get(page_no, (612.0, 792.0))
267 bbox = item.prov[0].bbox
268 if bbox.coord_origin == CoordOrigin.TOPLEFT:
269 box = (bbox.l, bbox.t, bbox.r, bbox.b)
270 else:
271 box = (bbox.l, height - bbox.t, bbox.r, height - bbox.b)
272 enclosing = [som for region, som in regions if _center_in(box, region)]
273 if not enclosing:
274 continue
275 covered = False
276 for som in enclosing:
277 bag = bags.get(id(som))
278 if bag is None:
279 bag = bags[id(som)] = _table_text_bag(som)
280 if _covered_by(item.text, bag):
281 covered = True
282 break
283 if covered:
284 victims.append(item)
285 else:
286 logger.info(
287 "Fusion: page {}: {} {} lies in the region of table {} but is not table text; kept: {!r}",
288 page_no,
289 item.label.value,
290 item.self_ref,
291 enclosing[0].table_id or enclosing[0].title,
292 " ".join(item.text.split()[:12]),
293 )
295 if victims:
296 unified.delete_items(node_items=victims)
299def _table_text_bag(table: ExtractedTable) -> Counter[str]:
300 """Every word the grafted table carries, with multiplicity: its cells, title,
301 subtitle, units, and footnote markers and bodies."""
302 parts = [table.title, table.subtitle, table.units]
303 for fn in table.footnotes:
304 parts.extend([fn.marker, fn.text])
305 for row in markdown_rows(table.markdown):
306 parts.extend(row)
307 return Counter(_TOKEN_RE.findall(" ".join(p for p in parts if p).lower()))
310def _covered_by(text: str, bag: Counter[str]) -> bool:
311 """Whether the table's words account for `text`: at most one token of it is
312 missing from the bag, or all but a rendering-noise fraction are present."""
313 tokens = _TOKEN_RE.findall(text.lower())
314 if not tokens:
315 return True
316 remaining = bag.copy()
317 matched = 0
318 for tok in tokens:
319 if remaining[tok] > 0:
320 remaining[tok] -= 1
321 matched += 1
322 unmatched = len(tokens) - matched
323 return unmatched <= _UNMATCHED_TOLERANCE or matched / len(tokens) >= _COVERAGE_MIN
326def _center_in(box: Box, region: Box) -> bool:
327 cx, cy = (box[0] + box[2]) / 2, (box[1] + box[3]) / 2
328 x0, y0 = min(region[0], region[2]), min(region[1], region[3])
329 x1, y1 = max(region[0], region[2]), max(region[1], region[3])
330 return x0 <= cx <= x1 and y0 <= cy <= y1
333def _apply_replace(
334 unified: DoclingDocument,
335 tables_by_ref: Dict[str, TableItem],
336 match: RegionMatch,
337 som_tables: List[ExtractedTable],
338 page_dims: Dict[int, Tuple[float, float]],
339) -> None:
340 """Rebuild each matched docling table's body from its paired Camelot table."""
341 for som_idx, ref in _pair_by_vertical_order(match, som_tables, tables_by_ref):
342 table_item = tables_by_ref.get(ref)
343 if table_item is None:
344 continue
345 som = som_tables[som_idx]
346 markdown = (som.markdown or "").strip()
347 if not markdown:
348 # A matched-but-empty Camelot body would blank a docling table that has
349 # OCR'd content; keep docling's body in that case.
350 continue
351 table_item.data = markdown_to_table_data(markdown)
352 attach_attribution(unified, table_item, som, page_dims)
355def _apply_insert(
356 unified: DoclingDocument,
357 match: RegionMatch,
358 som_tables: List[ExtractedTable],
359 page_dims: Dict[int, Tuple[float, float]],
360) -> Tuple[List[TableItem], List[str]]:
361 """Insert a Camelot table docling missed, at its reading-order position.
363 Returns the inserted tables and an error for any table that has no Camelot
364 body: a table SoM located but neither docling nor Camelot could read (a
365 scanned table on a page docling did not detect). It is surfaced, never
366 silently dropped.
367 """
368 tables: List[TableItem] = []
369 errors: List[str] = []
370 for som_idx in match.som_indices:
371 som = som_tables[som_idx]
372 markdown = (som.markdown or "").strip()
373 if not markdown:
374 errors.append(
375 f"page {som.page}: SoM located a table docling missed but Camelot read no "
376 f"body (title={som.title!r}); table not in the unified document"
377 )
378 continue
379 data = markdown_to_table_data(markdown)
380 prov = _som_prov(som, page_dims)
381 sibling, after = reading_order_anchor(unified, som, page_dims)
382 if sibling is None:
383 inserted = unified.add_table(data=data, prov=prov)
384 else:
385 inserted = unified.insert_table(sibling=sibling, data=data, prov=prov, after=after)
386 attach_attribution(unified, inserted, som, page_dims)
387 tables.append(inserted)
388 return tables, errors
391def _apply_undercount(
392 unified: DoclingDocument,
393 tables_by_ref: Dict[str, TableItem],
394 match: RegionMatch,
395 som_tables: List[ExtractedTable],
396 page_dims: Dict[int, Tuple[float, float]],
397) -> Tuple[List[TableItem], List[str]]:
398 """Delete the docling table(s) in the region and insert the finer Camelot tables.
400 docling under-counted (dropped or merged a complex table); Camelot's finer
401 count is trusted. The Camelot tables are inserted after the spine element
402 above the region, in vertical order. Returns the inserted tables and an
403 error for any Camelot table with no body, so a dropped sub-table is
404 surfaced rather than lost.
405 """
406 tables: List[TableItem] = []
407 errors: List[str] = []
408 doomed: List[NodeItem] = [tables_by_ref[r] for r in match.docling_table_refs if r in tables_by_ref]
409 anchor_som = min(
410 (som_tables[i] for i in match.som_indices),
411 key=lambda t: _vertical_key(t),
412 default=None,
413 )
414 sibling: Optional[NodeItem] = None
415 after = True
416 if anchor_som is not None:
417 sibling, after = reading_order_anchor(unified, anchor_som, page_dims, exclude=doomed)
419 if doomed:
420 unified.delete_items(node_items=doomed)
422 last: Optional[TableItem] = None
423 for som_idx in sorted(match.som_indices, key=lambda i: _vertical_key(som_tables[i])):
424 som = som_tables[som_idx]
425 markdown = (som.markdown or "").strip()
426 if not markdown:
427 errors.append(
428 f"page {som.page}: a sub-table of a docling-merged region had no Camelot body "
429 f"(title={som.title!r}); not in the unified document"
430 )
431 continue
432 data = markdown_to_table_data(markdown)
433 prov = _som_prov(som, page_dims)
434 if last is not None:
435 last = unified.insert_table(sibling=last, data=data, prov=prov, after=True)
436 elif sibling is not None:
437 last = unified.insert_table(sibling=sibling, data=data, prov=prov, after=after)
438 else:
439 last = unified.add_table(data=data, prov=prov)
440 attach_attribution(unified, last, som, page_dims)
441 tables.append(last)
442 return tables, errors
445def attach_attribution(
446 unified: DoclingDocument,
447 table_item: TableItem,
448 som: ExtractedTable,
449 page_dims: Dict[int, Tuple[float, float]],
450) -> None:
451 """Carry the SoM identity, cell geometry, title, subtitle, units (captions)
452 and footnotes onto a table.
454 The SoM table's `table_id` and `content_fingerprint` bind onto the docling
455 item through `meta` (docling's sanctioned extension point — the meta
456 model allows extra fields, so they serialize with the document), and every
457 grafted cell carries its measured box where the provenance record can pin
458 one. SoM is the vetted, vision-grounded side, so its attribution takes
459 precedence over whatever docling attached:
461 - the sentence that introduces the table, its printed name and its units
462 become CAPTION nodes referenced from the table (top-to-bottom order; see
463 `caption_lines`). The table serializer emits captions inline, so they
464 render above the table in markdown/HTML.
465 - each footnote is inserted as a FOOTNOTE node in the body right after the
466 table, in order. docling's markdown serializer does NOT emit a table's
467 `footnotes` refs, so a referenced footnote would round-trip in JSON but
468 vanish from the rendered markdown; a body node after the table renders as a
469 paragraph below it and keeps the reading order.
471 Both kinds carry no provenance box, so the region-text dedup that strips
472 docling's loose copies never matches them.
473 """
474 if som.table_id:
475 base = table_item.meta or FloatingMeta()
476 table_item.meta = base.model_copy(
477 update={
478 "quber__table_id": som.table_id,
479 "quber__content_fingerprint": som.content_fingerprint,
480 }
481 )
482 _graft_cell_geometry(table_item, som, page_dims)
484 caption_refs = []
485 for text in caption_lines(unified, som):
486 caption = unified.add_text(label=DocItemLabel.CAPTION, text=text)
487 caption_refs.append(caption.get_ref())
488 if caption_refs:
489 table_item.captions = caption_refs
491 anchor: NodeItem = table_item
492 for note in som.footnotes:
493 # Render the footnote the way the page prints it: its own marker,
494 # then its text. An unmarked general note has no marker to lead with.
495 text = f"{note.marker} {note.text}".strip() if note.marker else note.text
496 if not text.strip():
497 continue
498 anchor = unified.insert_text(label=DocItemLabel.FOOTNOTE, text=text, sibling=anchor, after=True)
501#: docling text items that can hold the sentence introducing a table.
502_CAPTION_SOURCE_LABELS = {
503 DocItemLabel.TEXT,
504 DocItemLabel.PARAGRAPH,
505 DocItemLabel.CAPTION,
506 DocItemLabel.SECTION_HEADER,
507}
510def caption_lines(unified: DoclingDocument, som: ExtractedTable) -> List[str]:
511 """The header lines a table carries, top to bottom: the sentence that
512 introduces it, its printed name, its units.
514 The introducing sentence is the docling paragraph on the table's page that
515 contains the caption the vetting agent copied. Two tables printed side by
516 side under one sentence then carry the same whole sentence, whichever words
517 each copy kept, and a bold lead-in the copy left out comes back with the
518 paragraph. When no such paragraph exists, because the sentence fell inside
519 the table region and the region dedup removed docling's copy, the copied
520 caption stands. A name that repeats the sentence is not written twice.
522 Page furniture is never a header line. A running header or footer the
523 parse labeled as such ("Table of Contents", the statement banner) is
524 printed on the page, so the printed-text check upstream lets it through
525 when the agent copies it; the parse's own label is what rules it out.
526 """
527 furniture = _furniture_keys(unified, som.page)
528 lines: List[str] = []
529 caption = (som.caption or "").strip()
530 if caption and printed_key(caption) not in furniture:
531 lines.append(_paragraph_containing(unified, som.page, caption) or caption)
532 title = (som.title or "").strip()
533 if title and printed_key(title) in furniture:
534 title = ""
535 if title and not (lines and printed_key(title) in printed_key(lines[0])):
536 lines.append(title)
537 # Artifacts from earlier runs still carry a subtitle; new runs leave it empty.
538 if som.subtitle and som.subtitle.strip():
539 lines.append(som.subtitle.strip())
540 if som.units and som.units.strip():
541 lines.append(som.units.strip())
542 return lines
545def _furniture_keys(unified: DoclingDocument, page: int) -> set[str]:
546 """The printed keys of the page's running headers and footers, as the parse labeled them."""
547 keys: set[str] = set()
548 for item in unified.texts:
549 if (
550 item.label in (DocItemLabel.PAGE_HEADER, DocItemLabel.PAGE_FOOTER)
551 and item.prov
552 and item.prov[0].page_no == page
553 ):
554 key = printed_key(item.text)
555 if key:
556 keys.add(key)
557 return keys
560def _paragraph_containing(unified: DoclingDocument, page: int, caption: str) -> Optional[str]:
561 """The text of the first docling item on `page` whose printed words contain
562 `caption`, whitespace collapsed; None when no item does. Items without a
563 page box are skipped, which excludes the caption nodes the graft adds."""
564 key = printed_key(caption)
565 for item in unified.texts:
566 if item.label not in _CAPTION_SOURCE_LABELS or not item.prov or item.prov[0].page_no != page:
567 continue
568 if key in printed_key(item.text):
569 return " ".join(item.text.split())
570 return None
573def markdown_to_table_data(markdown: str) -> TableData:
574 """Parse a pipe-delimited markdown table into a docling `TableData` grid.
576 The first content row is treated as the column header. The markdown separator
577 row (`| --- | --- |`) is dropped. Ragged rows are padded to the widest row so
578 the grid stays rectangular.
579 """
580 rows = markdown_rows(markdown)
581 if not rows:
582 return TableData(table_cells=[], num_rows=0, num_cols=0)
584 num_cols = max(len(r) for r in rows)
585 num_rows = len(rows)
586 table_cells: List[TableCell] = []
587 for r, row in enumerate(rows):
588 padded = row + [""] * (num_cols - len(row))
589 for c, text in enumerate(padded):
590 table_cells.append(
591 TableCell(
592 text=text,
593 start_row_offset_idx=r,
594 end_row_offset_idx=r + 1,
595 start_col_offset_idx=c,
596 end_col_offset_idx=c + 1,
597 row_span=1,
598 col_span=1,
599 column_header=(r == 0),
600 )
601 )
602 return TableData(table_cells=table_cells, num_rows=num_rows, num_cols=num_cols)
605def _graft_cell_geometry(
606 table_item: TableItem, som: ExtractedTable, page_dims: Dict[int, Tuple[float, float]]
607) -> None:
608 """Carry the already-resolved cell geometry onto the grafted TableCells.
610 `corrected_grid` is the complete cell-level view the grounding stage
611 produced — same markdown, same row/col addressing — so this is pure
612 carriage: read each cell's box and convert it from the normalized top-left
613 frame to docling's bottom-left points, the same convention `_som_prov`
614 uses for the table-level box. No matching or reconciliation happens here;
615 a cell the grounding stage could not pin arrives with box=None and stays
616 without one.
617 """
618 grid = som.corrected_grid
619 if not grid:
620 return
621 width, height = page_dims.get(som.page, (612.0, 792.0))
622 for cell in table_item.data.table_cells:
623 r, c = cell.start_row_offset_idx, cell.start_col_offset_idx
624 box = grid[r][c].box if r < len(grid) and c < len(grid[r]) else None
625 if box is None:
626 continue
627 x1, y1, x2, y2 = box
628 cell.bbox = BoundingBox(
629 l=min(x1, x2) * width,
630 r=max(x1, x2) * width,
631 t=(1.0 - min(y1, y2)) * height,
632 b=(1.0 - max(y1, y2)) * height,
633 coord_origin=CoordOrigin.BOTTOMLEFT,
634 )
637def _pair_by_vertical_order(
638 match: RegionMatch,
639 som_tables: List[ExtractedTable],
640 tables_by_ref: Dict[str, TableItem],
641) -> List[Tuple[int, str]]:
642 """Pair SoM tables to docling tables in a region by top-to-bottom order."""
643 soms = sorted(match.som_indices, key=lambda i: _vertical_key(som_tables[i]))
644 refs = sorted(
645 (r for r in match.docling_table_refs if r in tables_by_ref),
646 key=lambda r: _table_top(tables_by_ref[r]),
647 )
648 return list(zip(soms, refs, strict=False))
651def _vertical_key(table: ExtractedTable) -> float:
652 region = table.som_region
653 if region is not None:
654 return min(region[1], region[3])
655 if table.bbox is not None:
656 return -max(table.bbox[1], table.bbox[3]) # bottom-left points: larger y is higher
657 return 0.0
660def _table_top(table: TableItem) -> float:
661 if not table.prov:
662 return 0.0
663 bbox = table.prov[0].bbox
664 if bbox.coord_origin == CoordOrigin.TOPLEFT:
665 return bbox.t
666 return -max(bbox.t, bbox.b) # bottom-left: larger y is higher on the page
669def _som_prov(som: ExtractedTable, page_dims: Dict[int, Tuple[float, float]]) -> Optional[ProvenanceItem]:
670 """A ProvenanceItem for an inserted Camelot table, box in bottom-left points."""
671 width, height = page_dims.get(som.page, (612.0, 792.0))
672 region = som.som_region
673 if region is None:
674 return None
675 x1, y1, x2, y2 = region
676 bbox = BoundingBox(
677 l=min(x1, x2) * width,
678 r=max(x1, x2) * width,
679 t=(1.0 - min(y1, y2)) * height,
680 b=(1.0 - max(y1, y2)) * height,
681 coord_origin=CoordOrigin.BOTTOMLEFT,
682 )
683 return ProvenanceItem(page_no=som.page, bbox=bbox, charspan=(0, 0))
686def reading_order_anchor(
687 document: DoclingDocument,
688 som: ExtractedTable,
689 page_dims: Dict[int, Tuple[float, float]],
690 exclude: Optional[List[NodeItem]] = None,
691) -> Tuple[Optional[NodeItem], bool]:
692 """Find the spine element to anchor an inserted table to.
694 Returns (sibling, after). The sibling is the last element above the table on
695 its page, with after=True. When the table sits above every element on the
696 page, returns that page's first element with after=False. Returns (None, True)
697 when the page has no other anchorable element (caller falls back to add_table).
698 """
699 _width, height = page_dims.get(som.page, (612.0, 792.0))
700 region = som.som_region
701 if region is None:
702 return None, True
703 table_top = min(region[1], region[3]) * height # top-left points
705 excluded = {id(e) for e in (exclude or [])}
706 on_page: List[Tuple[float, NodeItem]] = []
707 for item, _level in document.iterate_items():
708 if not isinstance(item, DocItem) or id(item) in excluded:
709 continue
710 prov = item.prov[0] if item.prov else None
711 if prov is None or prov.page_no != som.page:
712 continue
713 top = _item_top_left_top(prov, height)
714 on_page.append((top, item))
716 if not on_page:
717 return None, True
719 on_page.sort(key=lambda x: x[0])
720 above = [item for top, item in on_page if top < table_top]
721 if above:
722 return above[-1], True
723 return on_page[0][1], False
726def _item_top_left_top(prov: ProvenanceItem, page_height: float) -> float:
727 bbox = prov.bbox
728 if bbox.coord_origin == CoordOrigin.TOPLEFT:
729 return bbox.t
730 return page_height - max(bbox.t, bbox.b)