Coverage for src / quber / core / figures / models.py: 95%

1"""What the figure workflow produces: one record per nominated page, and one per chart. 

2 

3A `FigureRun` is the whole output. It holds one `PageScan` for every page the 

4parse nominated — including the pages that were dropped before scanning and the 

5pages that were scanned and came back with no chart — so a page can never 

6disappear between nomination and output. A chart in a financial document is 

7often where a number appears that appears nowhere else, so a page that was 

8nominated and produced nothing has to be visible. 

9 

10A `FigureRecord` is one figure the scan returned. It carries the returned text 

11exactly as it arrived and the box the scan drew around it. The text is not 

12parsed into fields: a chart description has no fixed shape, and two bar charts 

13in one response used different field names in a different order, so a parser 

14written against what has been seen encodes one shape and misreads the next. 

15 

16A `ScannedTable` is one table the scan returned. A table comes back as a grid 

17with a box on every cell, so unlike a chart it is kept as structure rather than 

18as prose, and it goes on to the same correction and grounding steps every other 

19table in the document goes through. 

20 

21`ChartContext` is the other text the scanned page returned — the chart titles 

22and the notes printed at the foot of the page. It is kept as returned and 

23attached to no chart. One note commonly serves several charts on a page, and 

24matching a superscript to the note it points at is a separate step, the same 

25way it is for tables. 

26""" 

27 

28from __future__ import annotations 

29 

30from typing import Dict, List, Literal, Optional, Tuple 

31 

32from pydantic import BaseModel, Field 

33 

34#: Normalized box as the scan returns it: 0..1 with the page's top-left as 

35#: origin, keyed left/top/right/bottom. The page scanned was the real page, so 

36#: the box is already a source coordinate and needs no mapping back. 

37Box = Dict[str, float] 

38 

39 

40class FigureRecord(BaseModel): 

41 """One chart a page scan returned.""" 

42 

43 page: int = Field(description="1-based source page the chart is printed on") 

44 text: str = Field(description="The scan's reading of the chart, stored exactly as returned") 

45 box: Optional[Box] = Field(default=None, description="Normalized box around the chart on the page") 

46 chunk_id: Optional[str] = Field( 

47 default=None, description="The scan's own id for this figure, for tracing back to the response" 

48 ) 

49 job_id: Optional[str] = Field( 

50 default=None, description="The scan job that produced this record (see the page's PageScan)" 

51 ) 

52 picture_ref: Optional[str] = Field( 

53 default=None, 

54 description=( 

55 "The picture in the refined parse this record was grafted onto. Absent when the " 

56 "figure overlapped no picture, which the run reports as an error." 

57 ), 

58 ) 

59 

60 

61class ScannedTable(BaseModel): 

62 """One table a page scan returned, as a grid with a box on every cell. 

63 

64 A table comes back shaped like a table: an HTML grid with an id on every 

65 cell, and a box for each of those ids. That is a different shape from a 

66 chart, which comes back as a description with no internal locations at all, 

67 and it is the right shape for the element it belongs to. 

68 

69 The grid is dense — one entry per row and column, blank where a spanning 

70 cell covers a position — so it drops straight into the same correction and 

71 grounding steps every other table in the document goes through. 

72 """ 

73 

74 page: int = Field(description="1-based source page the table is printed on") 

75 cells: List[List[str]] = Field(description="The returned grid, rows of cell text") 

76 cell_boxes: List[List[Optional[Box]]] = Field( 

77 description="Normalized box per cell, shaped exactly like `cells`; absent where none was returned" 

78 ) 

79 box: Optional[Box] = Field(default=None, description="Normalized box around the whole table") 

80 chunk_id: Optional[str] = Field(default=None, description="The scan's own id for this table") 

81 job_id: Optional[str] = Field( 

82 default=None, description="The scan job that produced this record (see the page's PageScan)" 

83 ) 

84 table_ref: Optional[str] = Field( 

85 default=None, 

86 description=( 

87 "The table in the refined parse this grid replaced. Absent when the returned " 

88 "table overlapped no table in the parse, which the run reports as an error." 

89 ), 

90 ) 

91 picture_ref: Optional[str] = Field( 

92 default=None, 

93 description=( 

94 "The picture in the parse this grid was printed over, when the parse detected " 

95 "the region but filed it as a picture rather than a table. The grid becomes a " 

96 "new table beside that picture." 

97 ), 

98 ) 

99 table_id: Optional[str] = Field( 

100 default=None, description="The extracted table this grid produced, by its address" 

101 ) 

102 

103 

104class ChartContext(BaseModel): 

105 """A title or note the scanned page returned, kept as returned and unattached.""" 

106 

107 page: int = Field(description="1-based source page the text is printed on") 

108 kind: str = Field(description="The chunk type the scan assigned, as returned (text, marginalia)") 

109 text: str = Field(description="The text, stored exactly as returned") 

110 box: Optional[Box] = Field(default=None, description="Normalized box around the text on the page") 

111 chunk_id: Optional[str] = Field(default=None, description="The scan's own id for this chunk") 

112 

113 

114PageStatus = Literal["figures", "tables", "empty", "dropped"] 

115 

116 

117class PageScan(BaseModel): 

118 """One nominated page and what became of it. 

119 

120 `status` says which: `figures` for a scanned page that returned at least one 

121 figure, `tables` for one that returned no figure but at least one table, 

122 `empty` for a scanned page that returned neither, `dropped` for a page the 

123 filter answered no on, which is never scanned and so is never billed. 

124 

125 `figures` does not promise a chart. Nothing here filters on what a figure 

126 contains, and a page's logo comes back as a figure of its own, so a page can 

127 reach `figures` on the strength of its letterhead alone. 

128 

129 `reused` marks a page whose records come from a scan a previous run already 

130 paid for. `credits` is what that scan cost when it was made, so the two 

131 together separate what this run spent from what it read. 

132 """ 

133 

134 page: int = Field(description="1-based source page") 

135 status: PageStatus 

136 reason: str = Field(default="", description="Why a page was dropped; empty for a scanned page") 

137 picture_classes: List[str] = Field( 

138 default_factory=list, description="The picture classes that nominated this page" 

139 ) 

140 table_refs: List[str] = Field( 

141 default_factory=list, 

142 description="Tables in the parse that nominated this page, read off the page image", 

143 ) 

144 job_id: Optional[str] = Field(default=None, description="The scan job id, absent on a dropped page") 

145 model: Optional[str] = Field(default=None, description="The model requested for the scan") 

146 version: Optional[str] = Field(default=None, description="The model version the scan reported") 

147 credits: Optional[float] = Field(default=None, description="Credits the scan was billed when it was made") 

148 reused: bool = Field( 

149 default=False, description="True when the records come from a scan an earlier run paid for" 

150 ) 

151 response_artifact: Optional[str] = Field( 

152 default=None, description="Filename of the raw response this page's records came from" 

153 ) 

154 figures: List[FigureRecord] = Field(default_factory=list) 

155 tables: List[ScannedTable] = Field(default_factory=list) 

156 context: List[ChartContext] = Field(default_factory=list) 

157 

158 

159class RemovedText(BaseModel): 

160 """One text the run took out of the parse because a figure had been read over it. 

161 

162 Recorded so a completeness check can tell a deliberate removal from a silent 

163 loss. Both look the same from outside: text the page prints that the document 

164 no longer holds. The check reads the source PDF's own text layer, which is the 

165 layer this text came out of, so the same printed line is present in both 

166 readings and matches on its words and on where it sits. 

167 

168 The box is the normalized top-left frame the rest of the workflow states 

169 positions in, so no caller converts. Text alone would be too loose — a page 

170 printing `0.0%` as a gridline and again in a footnote gives one string for two 

171 places, and forgiving the gridline would forgive losing the footnote. 

172 """ 

173 

174 page: int = Field(description="The 1-based page it was printed on") 

175 text: str = Field(description="What it said, verbatim") 

176 box: Optional[Tuple[float, float, float, float]] = Field( 

177 default=None, description="Where it sat: (x1, y1, x2, y2), 0..1 from the page's top-left" 

178 ) 

179 reason: str = Field(default="", description="Why it was taken to be the figure's furniture") 

180 

181 

182class FigureValue(BaseModel): 

183 """One plotted value read off a figure, reconciled and traced to the page. 

184 

185 Two independent readers produce the inputs: the page scan's prose reading 

186 and a local read of the page image grounded in the parse's positioned text 

187 cells. A value both agree on, whose cited fragment prints it, is 

188 `reconciled` and carries that fragment's box. A positional disagreement is 

189 `value_misread`; a value only one reader produced is `value_unreconciled` 

190 with the direction in the note. Statuses come from the cell-status 

191 registry, so review surfaces treat figure values and table cells alike. 

192 """ 

193 

194 page: int = Field(description="1-based source page") 

195 picture_ref: Optional[str] = Field( 

196 default=None, description="The parse picture the value's figure was grafted onto" 

197 ) 

198 chart_title: str = Field(default="", description="The figure's printed title, as read") 

199 label: str = Field(description="Category or axis label the value belongs to") 

200 series: str = Field(default="", description="Series name when the figure has more than one") 

201 value: str = Field(description="The value as printed, including currency and sign marks") 

202 status: str = Field(description="A code from the cell-status registry") 

203 note: Optional[str] = Field(default=None, description="One line of evidence or direction") 

204 fragment_ids: List[str] = Field( 

205 default_factory=list, description="Ids of the page fragments grounding the value" 

206 ) 

207 box: Optional[Box] = Field(default=None, description="Normalized box of the fragment printing the value") 

208 

209 

210class FigureValueRun(BaseModel): 

211 """Every reconciled figure value one run produced, written as its own artifact.""" 

212 

213 document: str = Field(description="The source document's base name") 

214 values: List[FigureValue] = Field(default_factory=list) 

215 errors: List[str] = Field(default_factory=list) 

216 

217 @property 

218 def reconciled(self) -> int: 

219 return sum(1 for v in self.values if v.status == "reconciled") 

220 

221 @property 

222 def flagged(self) -> int: 

223 return sum(1 for v in self.values if v.status != "reconciled") 

224 

225 

226class FigureRun(BaseModel): 

227 """Everything one figure run produced, for the CLI to print and callers to read.""" 

228 

229 document: str = Field(description="The source document's base name") 

230 scans: List[PageScan] = Field(default_factory=list) 

231 errors: List[str] = Field( 

232 default_factory=list, 

233 description="Surfaced problems: a chart that matched no picture in the parse, a page whose scan failed", 

234 ) 

235 removed: List[RemovedText] = Field( 

236 default_factory=list, 

237 description="Text taken out of the parse as a read figure's furniture, so a completeness check can account for it", 

238 ) 

239 

240 @property 

241 def nominated(self) -> int: 

242 """Pages the parse nominated as holding a chart.""" 

243 return len(self.scans) 

244 

245 @property 

246 def scanned(self) -> int: 

247 """Nominated pages the run has a scan for, whether it paid for it or reused one.""" 

248 return sum(1 for s in self.scans if s.status != "dropped") 

249 

250 @property 

251 def submitted(self) -> int: 

252 """Pages this run submitted, and so was billed for.""" 

253 return sum(1 for s in self.scans if s.status != "dropped" and not s.reused) 

254 

255 @property 

256 def with_figures(self) -> int: 

257 """Scanned pages that returned at least one figure.""" 

258 return sum(1 for s in self.scans if s.status == "figures") 

259 

260 @property 

261 def with_tables(self) -> int: 

262 """Scanned pages that returned at least one table.""" 

263 return sum(1 for s in self.scans if s.tables) 

264 

265 @property 

266 def figures(self) -> List[FigureRecord]: 

267 """Every figure record the run produced, in page order.""" 

268 return [c for s in self.scans for c in s.figures] 

269 

270 @property 

271 def tables(self) -> List[ScannedTable]: 

272 """Every table the run read off a page image, in page order.""" 

273 return [t for s in self.scans for t in s.tables] 

274 

275 @property 

276 def credits(self) -> float: 

277 """Credits this run was billed. A reused scan was paid for by the run that made it.""" 

278 return sum(s.credits or 0.0 for s in self.scans if not s.reused)