"""Extract every inline-XBRL fact from the filing and measure how many of the
rendered fact texts survive into docling's markdown output."""

import json
from collections import Counter
from pathlib import Path

from lxml import html as lhtml

here = Path(__file__).parent
tree = lhtml.parse(str(here / "aapl-20260627.htm"))
root = tree.getroot()

IX = "ix:"
markdown = (here / "aapl-20260627.docling.md").read_text()

facts = []
hidden_facts = 0
hidden = list(root.iter("ix:hidden"))
hidden_nodes = set()
for h in hidden:
    for el in h.iter():
        hidden_nodes.add(el)
        if el.tag in ("ix:nonfraction", "ix:nonnumeric"):
            hidden_facts += 1

for tag in ("nonfraction", "nonnumeric"):
    for el in root.iter(f"ix:{tag}"):
        if el in hidden_nodes:
            continue
        text = "".join(el.itertext()).strip()
        facts.append(
            {
                "kind": tag,
                "name": el.get("name"),
                "text": text,
                "scale": el.get("scale"),
                "sign": el.get("sign"),
                "contextRef": el.get("contextref"),
                "unitRef": el.get("unitref"),
            }
        )

numeric = [f for f in facts if f["kind"] == "nonfraction" and f["text"]]
found = [f for f in numeric if f["text"] in markdown]
missing = [f for f in numeric if f["text"] not in markdown]

concepts = Counter(f["name"].split(":")[0] for f in facts)
report = {
    "visible_facts": len(facts),
    "hidden_facts": hidden_facts,
    "numeric_visible_nonempty": len(numeric),
    "numeric_text_found_in_markdown": len(found),
    "numeric_text_missing": len(missing),
    "missing_sample": missing[:10],
    "namespace_counts": dict(concepts),
    "scaled_facts": sum(1 for f in numeric if f["scale"] not in (None, "0")),
    "negated_sign_facts": sum(1 for f in numeric if f["sign"] == "-"),
}
(here / "fact_recall.json").write_text(json.dumps(report, indent=2))
(here / "facts.json").write_text(json.dumps(facts, indent=2))
print(json.dumps(report, indent=2)[:2500])