Coverage for src / quber / core / fusion / footnotes.py: 96%
141 statements
« prev ^ index » next coverage.py v7.14.0, created at 2026-09-23 22:14 -0400
1"""Resolve a table's footnote reference markers to their definition text.
3A marker printed on a table cell points at a footnote, but the footnote's
4text can be printed in three places, each needing its own resolution step:
61. Below the table, inside the correction agent's crop. The agent already
7 transcribed those as marker-and-text pairs (`FootnoteDef`), so pairing is
8 a normalized-key match — `(1)`, `1` and a superscript one are the same
9 marker. The crop ends where the page does, so a footnote that continues on
10 the next page reaches the agent cut mid-sentence; when the scan tier's line
11 for the same marker opens with everything the agent read and continues
12 past it, that longer line is taken as the definition instead.
132. Beyond the crop: later on the page or on a following page, including a
14 continuation block (markers `a` through `j` printed overleaf). Those are
15 found by scanning the document's text lines in reading order after the
16 table. Acceptance requires coherence: a matching line on the table's own
17 page is trusted (it sits right under the table), and so is a line inside
18 a run of consecutive marker-opened lines (the shape of a real
19 continuation block). A solitary line whose leading token happens to be a
20 digit, found on a distant page, proves nothing and is refused.
213. Nowhere as a footnote at all: a cross-reference to a NAMED SECTION of
22 the document — `(Note 12)`, `Schedule II`, `(Addendum 3)` — points at a
23 whole section, not a footnote line. WHICH markers are section references
24 is the correction agent's judgement, made from the page image and carried
25 on each mark's `kind`; nothing here infers it from the marker's wording.
26 A section reference resolves to a POINTER — the heading that opens with
27 the reference as written, plus its page — never to inlined text.
29Everything here is pure and deterministic. The demand-driven lookup agent
30(`quber.agents.footnote_lookup`) is the third tier for markers this module
31leaves unresolved; callers invoke it separately so the common case costs no
32model call. Every marker either resolves or lands in `unresolved`, and every
33marked definition either matches a marker or lands in `unreferenced` —
34nothing is silently dropped. Unmarked general notes are not exceptions: they
35carry no marker to match, and the consumer attaches them at table level.
36"""
38from __future__ import annotations
40import re
41from typing import Collection, List, Literal, Optional, Sequence, Tuple
43from pydantic import BaseModel, Field
45from quber.agents.llm_client import FootnoteDef
48class FootnoteResolution(BaseModel):
49 """The complete resolution for one table's markers, with its exceptions."""
51 resolved: List["ResolvedFootnote"] = Field(default_factory=list)
52 unresolved: List[str] = Field(
53 default_factory=list,
54 description="Markers (as printed) whose definition was not found in any tier",
55 )
56 unreferenced: List[FootnoteDef] = Field(
57 default_factory=list,
58 description="Marked definitions that no marker on the table points at",
59 )
61 def text_for(self, marker: str) -> Optional[str]:
62 """The resolved text for a printed marker, matched by canonical key."""
63 key = canonical_marker(marker)
64 for r in self.resolved:
65 if canonical_marker(r.marker) == key:
66 return r.text
67 return None
70class ResolvedFootnote(BaseModel):
71 """One marker paired with the text a reader should see for it."""
73 marker: str = Field(description="The marker as printed on the table")
74 text: str = Field(description="The definition text, or the pointer line for a Notes reference")
75 source: Literal["table", "scan", "note_pointer", "lookup", "sibling"] = Field(
76 description=(
77 "table: paired in the agent's own crop. scan: found by the reading-order "
78 "scan beyond the crop. note_pointer: a Notes-section reference resolved to "
79 "its heading and page. lookup: found by the demand-driven lookup agent. "
80 "sibling: a definition printed under a neighboring table of the same series."
81 )
82 )
85# Superscript glyphs normalize to their plain forms so a superscript one and a
86# printed '(1)' key the same.
87_SUPERSCRIPTS = str.maketrans("⁰¹²³⁴⁵⁶⁷⁸⁹ᵃᵇᶜᵈᵉᶠᵍʰⁱʲᵏ", "0123456789abcdefghijk")
90# The leading token of a printed footnote line. Digits and symbols may stand
91# bare ('2 Includes...'); letters must be set off by parentheses or trailing
92# punctuation ('a.' / '(a)'), because an ordinary prose line also starts with
93# a short word and would otherwise read as a marker.
94_LEAD_MARKER_RE = re.compile(
95 r"^(?:"
96 r"\((?P<paren>\d{1,2}|[A-Za-z]{1,2}|[*†‡§]{1,3})\)"
97 r"|(?P<bare>\d{1,2}|[*†‡§]{1,3})"
98 r"|(?P<punct>[A-Za-z]{1,2})[.):]"
99 r")\s+"
100)
103def canonical_marker(marker: str) -> str:
104 """A marker's identity independent of rendering: '(1)', '1' and a
105 superscript one all key to '1'; letters casefold."""
106 s = re.sub(r"\s+", "", marker or "").translate(_SUPERSCRIPTS)
107 return s.strip("()[]").rstrip(".:").casefold()
110def split_leading_marker(line: str) -> Optional[Tuple[str, str]]:
111 """(canonical marker, rest of line) when `line` opens the way a printed
112 footnote does; None otherwise. The rest must be non-empty — a marker
113 with nothing after it defines nothing."""
114 m = _LEAD_MARKER_RE.match(line.strip())
115 if not m:
116 return None
117 token = m.group("paren") or m.group("bare") or m.group("punct")
118 rest = line.strip()[m.end() :].strip()
119 if not rest:
120 return None
121 return canonical_marker(token), rest
124def resolve_footnotes(
125 markers: Sequence[str],
126 footnotes: Sequence[FootnoteDef],
127 table_page: int,
128 trailing_lines: Sequence[Tuple[str, int]] = (),
129 headings: Sequence[Tuple[str, int]] = (),
130 section_keys: Collection[str] = frozenset(),
131) -> FootnoteResolution:
132 """Resolve every distinct marker on one table, in tier order.
134 `markers` are the table's printed reference markers (the located marks
135 plus the catalogued refs). `footnotes` are the agent's in-crop pairs.
136 `trailing_lines` are the document's text lines in reading order after the
137 table, each with its 1-indexed page — the scan tier's universe.
138 `headings` are the document's section headings with their pages.
139 `section_keys` are the canonical keys of the markers the correction agent
140 judged to be cross-references to a named section of the document — that
141 judgement is the agent's, made from the page image; nothing here infers
142 it from the marker's wording or shape. A section reference resolves to a
143 heading pointer or stays unresolved; it is never hunted as a footnote.
144 Returns the resolution plus both exception lists; markers still
145 unresolved here are the lookup agent's demand.
146 """
147 defs_by_key: dict[str, FootnoteDef] = {}
148 for d in footnotes:
149 key = canonical_marker(d.marker)
150 if key:
151 defs_by_key.setdefault(key, d)
153 parsed_lines = [(split_leading_marker(text), page) for text, page in trailing_lines]
154 # Runs of consecutive marker-opened lines: the shape of a continuation
155 # block. block_len[i] is the length of the run line i belongs to.
156 block_len = [0] * len(parsed_lines)
157 i = 0
158 while i < len(parsed_lines):
159 if parsed_lines[i][0] is None:
160 i += 1
161 continue
162 j = i
163 while j < len(parsed_lines) and parsed_lines[j][0] is not None:
164 j += 1
165 for k in range(i, j):
166 block_len[k] = j - i
167 i = j
169 resolved: List[ResolvedFootnote] = []
170 unresolved: List[str] = []
171 seen_keys: set[str] = set()
172 for printed in markers:
173 key = canonical_marker(printed)
174 if not key or key in seen_keys:
175 continue
176 seen_keys.add(key)
178 if key in section_keys:
179 pointer = _section_pointer(printed, headings)
180 if pointer:
181 resolved.append(ResolvedFootnote(marker=printed, text=pointer, source="note_pointer"))
182 else:
183 unresolved.append(printed)
184 continue
186 candidates = _scan_candidates(key, parsed_lines, block_len, table_page)
188 if key in defs_by_key:
189 in_crop = defs_by_key[key].text
190 # The crop ends where the page does. A footnote whose text runs on
191 # to the next page reaches the agent cut mid-sentence, while the
192 # parse stitches both pages into one line. When a line the scan
193 # accepts for the same marker opens with everything the agent read
194 # and continues past it, that line is the complete definition. The
195 # agent's own text also rides in the scan universe (the graft
196 # attaches it to the table), so the first match is often the same
197 # cut text; every accepted line is tried.
198 longer = next((c for c in candidates if _extends(in_crop, c)), None)
199 if longer is not None:
200 resolved.append(ResolvedFootnote(marker=printed, text=longer, source="scan"))
201 else:
202 resolved.append(ResolvedFootnote(marker=printed, text=in_crop, source="table"))
203 continue
205 if candidates:
206 resolved.append(ResolvedFootnote(marker=printed, text=candidates[0], source="scan"))
207 continue
209 # A marker the agent judged a footnote but that no tier defines can
210 # still be a mislabelled section reference. The test is the same one
211 # section references pass: a document heading literally opens with
212 # the printed marker. Restricted to word-plus-number markers
213 # ('Note 18') — a bare '(1)' or '(a)' can never fall through to a
214 # heading, so a genuine footnote marker with a missing definition
215 # stays unresolved and flagged.
216 if (
217 any(ch.isalpha() for ch in printed)
218 and any(ch.isdigit() for ch in printed)
219 and (pointer := _section_pointer(printed, headings))
220 ):
221 resolved.append(ResolvedFootnote(marker=printed, text=pointer, source="note_pointer"))
222 else:
223 unresolved.append(printed)
225 unreferenced = [
226 d for d in footnotes if canonical_marker(d.marker) and canonical_marker(d.marker) not in seen_keys
227 ]
228 return FootnoteResolution(resolved=resolved, unresolved=unresolved, unreferenced=unreferenced)
231def absorb_lookup(
232 resolution: FootnoteResolution,
233 found: Sequence[FootnoteDef],
234 source: Literal["lookup", "sibling"] = "lookup",
235) -> FootnoteResolution:
236 """Fold externally found definitions into a resolution.
238 Each found pair matching a still-unresolved marker resolves it under
239 `source`; markers not found stay unresolved, and the rest of the
240 resolution is unchanged."""
241 by_key = {canonical_marker(d.marker): d for d in found if canonical_marker(d.marker)}
242 resolved = list(resolution.resolved)
243 unresolved: List[str] = []
244 for printed in resolution.unresolved:
245 d = by_key.get(canonical_marker(printed))
246 if d is not None:
247 resolved.append(ResolvedFootnote(marker=printed, text=d.text, source=source))
248 else:
249 unresolved.append(printed)
250 return FootnoteResolution(
251 resolved=resolved, unresolved=unresolved, unreferenced=list(resolution.unreferenced)
252 )
255def _extends(shorter: str, longer: str) -> bool:
256 """Whether `longer` opens with the whole of `shorter` and continues past
257 it, compared word by word with case, spacing and punctuation dropped so
258 the two extractors' renderings of the same printed line agree. A hyphen
259 dropped at a line break ('interest-only' against 'interestonly') is the
260 remaining difference, so the comparison also joins the words."""
261 a = _words(shorter)
262 b = _words(longer)
263 if not a or len(b) <= len(a):
264 return False
265 if b[: len(a)] == a:
266 return True
267 joined_a = "".join(a)
268 joined_b = "".join(b)
269 return len(joined_b) > len(joined_a) and joined_b.startswith(joined_a)
272def _words(text: str) -> List[str]:
273 return re.findall(r"[a-z0-9]+", text.casefold())
276def _scan_candidates(
277 key: str,
278 parsed_lines: Sequence[Tuple[Optional[Tuple[str, str]], int]],
279 block_len: Sequence[int],
280 table_page: int,
281) -> List[str]:
282 """The scan tier: every coherent line opening with `key`, in reading order.
284 Coherent means the line is on the table's own page (it sits right under
285 the table), or it belongs to a run of two or more consecutive
286 marker-opened lines (a continuation block), or its marker is not purely
287 numeric (a bare digit is the one leading token common prose also
288 produces). The remaining case — a solitary numeric match on a distant
289 page — is refused. The first entry is the scan tier's own answer; the rest
290 matter only when the agent's in-crop text is being checked for a longer
291 rendering of the same footnote.
292 """
293 found: List[str] = []
294 for (parsed, page), run in zip(parsed_lines, block_len, strict=True):
295 if parsed is None or parsed[0] != key:
296 continue
297 if page == table_page or run >= 2 or not key.isdigit():
298 found.append(parsed[1])
299 return found
302def _section_pointer(printed: str, headings: Sequence[Tuple[str, int]]) -> Optional[str]:
303 """The pointer line for a section reference: the target's heading and page.
305 The reference as the agent wrote it, normalized (spacing, punctuation,
306 case dropped), must be the opening of a heading — 'Note 16' opens
307 'NOTE 16. Commitments and Contingencies', 'Schedule II' opens
308 'Schedule II — Valuation and Qualifying Accounts'. Two printed forms of
309 the same target also agree: a reference written with a word where the
310 heading opens with the bare number (or the reverse) matches on the
311 number — 'Note 10' points at '10. Contingencies' — but only when the
312 reference itself is a word-plus-number, never for a bare digit, and only
313 at a digit boundary so '10' cannot open '104'. A reference naming
314 several targets ('Notes 1 and 4') resolves when every number it names
315 finds a heading; the pointer then lists them all. None when no heading
316 matches — the reference then stays unresolved and flagged rather than
317 pointing at nothing."""
318 key = _heading_key(printed)
319 if not key:
320 return None
321 for text, page in headings:
322 if _heading_key(text).startswith(key):
323 return f"See {text.strip()} (page {page})"
325 lead = re.match(r"([a-z]+)\d", key)
326 if not lead:
327 return None
328 ref_word = lead.group(1)
329 pointers = []
330 for number in re.findall(r"\d+", key):
331 hit = None
332 for text, page in headings:
333 m = re.match(r"([a-z]*)(\d+)", _heading_key(text))
334 if m is None or m.group(2) != number:
335 continue
336 hword = m.group(1)
337 if hword and not (hword.startswith(ref_word) or ref_word.startswith(hword)):
338 continue
339 hit = f"See {text.strip()} (page {page})"
340 break
341 if hit is None:
342 return None
343 pointers.append(hit)
344 return "; ".join(pointers) if pointers else None
347def _heading_key(text: str) -> str:
348 """Text reduced to its letters and digits, casefolded — the form in which
349 a printed reference and its target heading agree regardless of spacing or
350 punctuation."""
351 return "".join(ch for ch in text if ch.isalnum()).casefold()