Coverage for src / quber / core / validate.py: 93%

1""" 

2Validation hooks — internal sanity checks. NOT an externally-presentable 

3audit (per plan §7). 

4 

5The load-bearing check is `camelot_vs_llm_count`: Camelot's detected 

6table count per page versus the LLM's image-based count. Mismatches 

7surface explicitly — financial-document liability framing means we 

8**do not silently judge** which extractor is right. The user's standing 

9instruction: "If this was a financial document and the critical items 

10happened to be on the failed page the legal liability falls on us. We 

11have no defense." 

12""" 

13 

14from __future__ import annotations 

15 

16from collections import Counter 

17from pathlib import Path 

18from typing import Dict, List, Optional 

19 

20from pydantic import BaseModel, Field 

21 

22from quber.agents.llm_client import LLMClient 

23from quber.core.extractors.base import ExtractedTable 

24 

25 

26class CountMismatch(BaseModel): 

27 page: int 

28 camelot_count: int 

29 llm_count: int 

30 

31 @property 

32 def delta(self) -> int: 

33 return self.llm_count - self.camelot_count 

34 

35 

36class ValidationReport(BaseModel): 

37 source: str 

38 total_pages: int 

39 pages_checked: int 

40 mismatches: List[CountMismatch] = Field(default_factory=list) 

41 errors: List[str] = Field(default_factory=list) 

42 

43 @property 

44 def has_mismatches(self) -> bool: 

45 return len(self.mismatches) > 0 

46 

47 

48def camelot_counts_by_page(tables: List[ExtractedTable]) -> Dict[int, int]: 

49 return dict(Counter(t.page for t in tables)) 

50 

51 

52async def camelot_vs_llm_count( 

53 source: Path, 

54 tables: List[ExtractedTable], 

55 page_images: List[Path], 

56 llm: LLMClient, 

57 pages: Optional[List[int]] = None, 

58 max_concurrent: int = 5, 

59) -> ValidationReport: 

60 """Per-page LLM count calls run concurrently (capped by `max_concurrent`). 

61 

62 Each page's count is independent, so they parallelize cleanly. 

63 """ 

64 import asyncio 

65 

66 camelot_counts = camelot_counts_by_page(tables) 

67 total_pages = len(page_images) 

68 pages_to_check = pages or list(range(1, total_pages + 1)) 

69 report = ValidationReport(source=str(source), total_pages=total_pages, pages_checked=len(pages_to_check)) 

70 

71 valid_pages = [p for p in pages_to_check if 1 <= p <= total_pages] 

72 for p in pages_to_check: 

73 if not (1 <= p <= total_pages): 

74 report.errors.append(f"page {p} out of range (1..{total_pages})") 

75 

76 semaphore = asyncio.Semaphore(max_concurrent) 

77 

78 async def count_one(page: int): 

79 image_path = page_images[page - 1] 

80 async with semaphore: 

81 try: 

82 return page, await llm.count_tables(image_path), None 

83 except Exception as exc: 

84 return page, None, exc 

85 

86 results = await asyncio.gather(*(count_one(p) for p in valid_pages)) 

87 for page, llm_count, exc in results: 

88 if exc is not None: 

89 report.errors.append(f"page {page}: llm.count_tables failed: {exc}") 

90 continue 

91 assert llm_count is not None 

92 camelot_count = camelot_counts.get(page, 0) 

93 if camelot_count != llm_count: 

94 report.mismatches.append( 

95 CountMismatch(page=page, camelot_count=camelot_count, llm_count=llm_count) 

96 ) 

97 

98 return report 

99 

100 

101def camelot_vs_llm_count_sync( 

102 source: Path, 

103 tables: List[ExtractedTable], 

104 page_images: List[Path], 

105 llm: LLMClient, 

106 pages: Optional[List[int]] = None, 

107 max_concurrent: int = 5, 

108) -> ValidationReport: 

109 """Sync entry point — wraps the async version with `asyncio.run`.""" 

110 import asyncio 

111 

112 return asyncio.run(camelot_vs_llm_count(source, tables, page_images, llm, pages, max_concurrent))