Coverage for src / quber / core / extractors / set_of_mark / extent.py: 100%

1"""Revise each located table's box to the true end of its content. 

2 

3The Set-of-Mark locator and Camelot both bound a table generously and can run 

4their box past the last data row, enclosing footnote lines printed beneath the 

5grid. The corrected markdown, by contrast, ends at the last tabular row. Matching 

6that last row's tokens back to the page text layer locates the real bottom, so 

7`content_region` is `som_region` with its bottom pulled up to it. 

8 

9`apply_content_regions` sets `content_region` on every Set-of-Mark table, reading 

10each page's text layer once. It only ever pulls the bottom UP to the located last 

11row; when the markdown is empty (no grid) or the row cannot be located it leaves 

12`content_region` equal to `som_region`, never inventing a tighter box. 

13""" 

14 

15from __future__ import annotations 

16 

17import re 

18from collections import defaultdict 

19from pathlib import Path 

20from typing import Dict, List, Sequence, Tuple 

21 

22from quber.agents.completeness import page_words 

23from quber.core.extractors.base import ExtractedTable 

24 

25Region = Tuple[float, float, float, float] 

26Word = Tuple[float, float, float, float, str] 

27 

28 

29def normalize(token: str) -> str: 

30 """A token reduced to lowercase alphanumerics, for text-layer matching.""" 

31 return re.sub(r"[^a-z0-9]", "", token.lower()) 

32 

33 

34def last_row_tokens(markdown: str) -> set[str]: 

35 """Distinctive tokens of the markdown's last data row (two chars or more). 

36 

37 The last row carries the table's bottom edge; its label plus values are 

38 matched against the page text to find where that row sits. 

39 """ 

40 rows = [ 

41 line 

42 for line in markdown.splitlines() 

43 if line.strip().startswith("|") and not set(line.strip()) <= set("|-: ") 

44 ] 

45 if not rows: 

46 return set() 

47 toks = {normalize(w) for cell in rows[-1].split("|") for w in cell.split()} 

48 return {t for t in toks if len(t) >= 2} 

49 

50 

51def revise_content_region( 

52 som_region: Region, markdown: str, words: Sequence[Word], page_w: float, page_h: float 

53) -> Region: 

54 """`som_region` with the bottom pulled up to the table's last data row. 

55 

56 The last row's tokens are matched to page words inside som_region's own x/y 

57 band, so a stacked sibling's identical last row is never picked up, and the 

58 bottom of the matched words becomes the new bottom edge. Returns `som_region` 

59 unchanged when the markdown has no rows or the row cannot be located. 

60 """ 

61 tokens = last_row_tokens(markdown) 

62 if not tokens: 

63 return som_region 

64 x0, y0, x1, y1 = som_region 

65 lo_x, hi_x = min(x0, x1) * page_w, max(x0, x1) * page_w 

66 lo_y, hi_y = min(y0, y1) * page_h, max(y0, y1) * page_h 

67 bottoms = [ 

68 w[3] 

69 for w in words 

70 if lo_x - 3 <= (w[0] + w[2]) / 2 <= hi_x + 3 

71 and lo_y - 3 <= (w[1] + w[3]) / 2 <= hi_y + 3 

72 and normalize(w[4]) in tokens 

73 ] 

74 if not bottoms: 

75 return som_region 

76 return (x0, y0, x1, max(bottoms) / page_h) 

77 

78 

79def apply_content_regions(tables: List[ExtractedTable], source: str) -> None: 

80 """Set `content_region` on every located table. Mutates in place. 

81 

82 Reads each page's text layer once. A table with no `som_region` (no locator) 

83 is left untouched; otherwise `content_region` is the located region with its 

84 bottom revised to the last tabular row, or the located region itself when no 

85 revision can be made. 

86 """ 

87 by_page: Dict[int, List[ExtractedTable]] = defaultdict(list) 

88 for table in tables: 

89 if table.som_region is not None: 

90 by_page[table.page].append(table) 

91 

92 for page, page_tables in by_page.items(): 

93 page_w, page_h, words = page_words(Path(source), page) 

94 for table in page_tables: 

95 assert table.som_region is not None # filtered above 

96 table.content_region = revise_content_region( 

97 table.som_region, table.markdown, words, page_w, page_h 

98 )