"""QUE-259 discovery: compare Set-of-Mark located tables against DoclingDocument picture/table items to find an observable signal that flags image content (charts mistaken as tables, and real tables rendered as images). Discovery only. Reads: docling/.docling.json (DoclingDocument: pictures[], tables[], pages{}) tables/.tables.json (list[ExtractedTable]: page, bbox, som_region, ...) Emits: out/.compare.json per-located-table correspondence out/findings.json aggregate out/findings.md readable report All boxes are normalized to TOP-LEFT 0..1 before overlap is computed. """ from __future__ import annotations import json from glob import glob from pathlib import Path HERE = Path(__file__).parent DOCLING = HERE / "docling" TABLES = HERE / "tables" OUT = HERE / "out" OUT.mkdir(exist_ok=True) # docling DocumentFigureClassifier classes that mean "this picture is a graphic # that is NOT a faithful data table" -- a chart/plot. Anything matching these # substrings is treated as chart-like. CHART_TOKENS = ("chart", "plot", "histogram", "heatmap", "graph", "diagram") # classes that are decorative / not data at all NONDATA_TOKENS = ("logo", "icon", "signature", "stamp") def norm_from_bottomleft(bbox: dict, w: float, h: float) -> tuple[float, float, float, float]: """docling bbox {l,t,r,b, coord_origin} -> normalized TOP-LEFT (x1,y1,x2,y2).""" l, t, r, b = bbox["l"], bbox["t"], bbox["r"], bbox["b"] if bbox.get("coord_origin", "BOTTOMLEFT") == "BOTTOMLEFT": y_top, y_bot = (h - t) / h, (h - b) / h else: # TOPLEFT already y_top, y_bot = t / h, b / h return (l / w, min(y_top, y_bot), r / w, max(y_top, y_bot)) def overlap(a, b) -> dict: ix1, iy1 = max(a[0], b[0]), max(a[1], b[1]) ix2, iy2 = min(a[2], b[2]), min(a[3], b[3]) inter = max(0.0, ix2 - ix1) * max(0.0, iy2 - iy1) aa = max(1e-9, (a[2] - a[0]) * (a[3] - a[1])) ba = max(1e-9, (b[2] - b[0]) * (b[3] - b[1])) return { "iou": inter / (aa + ba - inter) if (aa + ba - inter) > 0 else 0.0, "cov_of_a": inter / aa, # how much of the located table is covered "cov_of_b": inter / ba, # how much of the docling item is covered } def page_size(pages, page_no) -> tuple[float, float]: p = pages.get(str(page_no)) or pages.get(page_no) if p is None: for v in pages.values(): if v.get("page_no") == page_no: p = v break sz = p["size"] return sz["width"], sz["height"] def chart_class(classes) -> tuple[str, float] | None: if not classes: return None top = max(classes, key=lambda c: c.get("confidence", 0)) return top.get("class_name"), round(top.get("confidence", 0), 3) def load_docling(doc_id): d = json.load(open(DOCLING / f"{doc_id}.docling.json")) pages = d["pages"] pics, tabs = [], [] for pic in d.get("pictures", []): if not pic.get("prov"): continue prov = pic["prov"][0] pg = prov["page_no"] w, h = page_size(pages, pg) cls = None for a in pic.get("annotations", []): if "predicted_classes" in a: cls = chart_class(a["predicted_classes"]) pics.append({"page": pg, "box": norm_from_bottomleft(prov["bbox"], w, h), "cls": cls}) for tab in d.get("tables", []): if not tab.get("prov"): continue prov = tab["prov"][0] pg = prov["page_no"] w, h = page_size(pages, pg) tabs.append({"page": pg, "box": norm_from_bottomleft(prov["bbox"], w, h)}) return pics, tabs, len(pages) def is_chart(cls) -> bool: return bool(cls and any(tok in cls[0] for tok in CHART_TOKENS)) def is_nondata(cls) -> bool: return bool(cls and any(tok in cls[0] for tok in NONDATA_TOKENS)) def analyze_doc(doc_id): pics, tabs, n_pages = load_docling(doc_id) som_tables = json.load(open(TABLES / f"{doc_id}.tables.json")) rows = [] for i, st in enumerate(som_tables): pg = st["page"] box = tuple(st["som_region"]) if st.get("som_region") else None acc = st.get("camelot_accuracy", 0.0) empty = not (st.get("markdown") or "").strip() rec = { "som_idx": i, "page": pg, "camelot_accuracy": acc, "empty_markdown": empty, "flavor": st.get("flavor"), "best_docling_table": None, "best_docling_picture": None, } if box: # best overlapping docling table on same page cand = [(overlap(box, t["box"]), t) for t in tabs if t["page"] == pg] if cand: ov, t = max(cand, key=lambda c: c[0]["cov_of_a"]) rec["best_docling_table"] = {"iou": round(ov["iou"], 3), "cov_som": round(ov["cov_of_a"], 3), "cov_item": round(ov["cov_of_b"], 3)} candp = [(overlap(box, p["box"]), p) for p in pics if p["page"] == pg] if candp: ov, p = max(candp, key=lambda c: c[0]["cov_of_a"]) rec["best_docling_picture"] = {"iou": round(ov["iou"], 3), "cov_som": round(ov["cov_of_a"], 3), "cov_item": round(ov["cov_of_b"], 3), "cls": p["cls"]} # verdict bt, bp = rec["best_docling_table"], rec["best_docling_picture"] t_cov = bt["cov_som"] if bt else 0.0 p_cov = bp["cov_som"] if bp else 0.0 p_cls = bp["cls"] if bp else None verdict = "table_ok" if empty or acc <= 1.0: verdict = "image_table_suspect" # camelot got nothing if p_cov >= 0.5 and p_cov > t_cov and is_chart(p_cls): verdict = "chart_as_table" elif p_cov >= 0.5 and p_cov > t_cov and not is_nondata(p_cls) and t_cov < 0.3: verdict = "image_region_suspect" # sits on a picture, not a docling table rec["verdict"] = verdict rows.append(rec) # docling charts that exist on the page, and whether a SoM table sat on them som_boxes = [(st["page"], tuple(st["som_region"])) for st in som_tables if st.get("som_region")] charts = [] for p in pics: if not is_chart(p["cls"]): continue grabbed = any(pg == p["page"] and overlap(p["box"], b)["cov_of_a"] >= 0.4 for pg, b in som_boxes) charts.append({"page": p["page"], "cls": p["cls"], "grabbed_as_table": grabbed}) return { "doc_id": doc_id, "n_pages": n_pages, "n_docling_tables": len(tabs), "n_docling_pictures": len(pics), "n_docling_charts": len(charts), "n_som_tables": len(som_tables), "rows": rows, "charts": charts, } def main(): ids = sorted(p.stem.replace(".docling", "") for p in DOCLING.glob("*.docling.json")) ids = [i for i in ids if (TABLES / f"{i}.tables.json").exists()] docs = [] for doc_id in ids: try: res = analyze_doc(doc_id) except Exception as e: # noqa: BLE001 res = {"doc_id": doc_id, "error": repr(e)} json.dump(res, open(OUT / f"{doc_id}.compare.json", "w"), indent=1) docs.append(res) json.dump(docs, open(OUT / "findings.json", "w"), indent=1) # readable report lines = ["# QUE-259 docling-vs-Set-of-Mark comparison", ""] tot_chart_as_table = tot_img_suspect = tot_charts = tot_charts_grabbed = 0 for d in docs: if d.get("error"): lines.append(f"## {d['doc_id']} -- ERROR: {d['error']}") continue flagged = [r for r in d["rows"] if r["verdict"] != "table_ok"] tot_chart_as_table += sum(r["verdict"] == "chart_as_table" for r in d["rows"]) tot_img_suspect += sum(r["verdict"] in ("image_table_suspect", "image_region_suspect") for r in d["rows"]) tot_charts += d["n_docling_charts"] tot_charts_grabbed += sum(c["grabbed_as_table"] for c in d["charts"]) lines.append( f"## {d['doc_id']} pages={d['n_pages']} " f"docling[tables={d['n_docling_tables']} pics={d['n_docling_pictures']} charts={d['n_docling_charts']}] " f"SoM_tables={d['n_som_tables']} flagged={len(flagged)}" ) for r in flagged: bp = r["best_docling_picture"] bt = r["best_docling_table"] lines.append( f" - p{r['page']} [{r['verdict']}] acc={r['camelot_accuracy']} empty={r['empty_markdown']} " f"| pic={bp and (bp['cls'], 'cov', bp['cov_som'])} | tbl_cov={bt and bt['cov_som']}" ) for c in d["charts"]: if c["grabbed_as_table"]: lines.append(f" - p{c['page']} CHART {c['cls']} was grabbed as a SoM table") lines.append("") lines.insert(1, ( f"Totals: chart_as_table={tot_chart_as_table} image_suspect={tot_img_suspect} " f"docling_charts={tot_charts} charts_grabbed_as_table={tot_charts_grabbed}\n" )) (OUT / "findings.md").write_text("\n".join(lines)) print("\n".join(lines)) if __name__ == "__main__": main()