"""What a batch exports: one row per question, as a workbook or as text. The two writers share the row building and nothing else, because each writes a different thing. A workbook cell carries its own type and number format, so the printed figure is a text cell nothing reformats and a percentage arrives as the fraction under a percent format, displaying at the printed scale. A text file encodes what it can in the characters: the printed column is quoted, the numeric column is not, and a percentage keeps its percent sign for the importer to divide. Both land the same value in the same cell; they need different bytes to get there. Every figure is written twice, as printed and as a number. Both are read off the answer — `value` and the derived `number` — and nothing here parses a printed figure. `answers/figures.py` is the one place that does, and the two sign-flipping defects that motivated that rule were both written by an export re-deriving what the answer already carried. A question with no answer is a third kind of row, distinct from a figure and from a failure: a sentinel in the printed column, an empty numeric column, and the model's explanation in the note. Formula injection is handled structurally. Workbook text cells are typed as text, so a question beginning with `=` is characters. In the CSV the free-text columns are prefixed, and only those — a printed figure is never prefixed, because that column has to match the filing byte for byte. """ from __future__ import annotations import json from decimal import Decimal from io import BytesIO from typing import Any, Dict, List, Optional import openpyxl from pydantic import BaseModel from quber.playground.answers.figures import is_percent, norm_printed from quber.playground.batch import RunState COLUMNS = [ "question", "as printed", "number", "unit", "period", "page", "cited ids", "source cell text", "status", "note", "review flag", "error", ] #: The printed-column sentinel for an `unanswerable` answer: retrieval ran, #: the model answered, and the answer is that the document does not print the #: figure. A reader sorting the sheet can then separate a figure, a figure the #: document does not contain, and a question that never got that far. The #: hashes mark it as ours: it is written off the answer's `unanswerable` #: payload kind, never taken from model text, so nothing a model writes and #: nothing a filing prints can collide with it. NOT_IN_DOCUMENT = "#not found#" #: What the review-flag column holds when a cited cell's status is registered #: for review. A word rather than a boolean so the filter reads as what it is. REVIEW = "review" _JOIN = "; " class ExportRow(BaseModel): """One question's row, typed for the writers rather than for a reader.""" question: str printed: str = "" number: Optional[Decimal] = None percent: bool = False # True when `printed` is a figure quoted verbatim from the filing — the # one kind of cell the CSV writer must never prefix. figure: bool = False unit: str = "" period: str = "" pages: str = "" cited_ids: str = "" cell_text: str = "" status: str = "" note: str = "" review: bool = False error: str = "" def build_rows(state: RunState) -> List[ExportRow]: return [_build_row(row.question, row.error, row.response) for row in state.rows] def _build_row(question: str, error: Optional[str], response: Optional[Dict[str, Any]]) -> ExportRow: # A question that failed carries its error and nothing else. if response is None: return ExportRow(question=question, error=error or "failed") row = ExportRow(question=question) refs = response.get("references") or [] payload = response.get("value") or {} kind = response.get("shape") if kind == "scalar": row.printed = payload.get("value") or "" row.figure = True row.percent = is_percent(row.printed) # `number` was derived by the answer; this is deserialization, not a # second reading of the printed figure. serialized = payload.get("number") row.number = Decimal(serialized) if serialized is not None else None row.unit = payload.get("unit") or "" row.period = payload.get("period") or "" elif kind == "unanswerable": row.printed = NOT_IN_DOCUMENT row.note = payload.get("reason") or "" elif kind == "prose": row.printed = payload.get("text") or "" else: # A series or grid has no single printed figure; the payload itself is # the answer, carried whole rather than truncated to a first point. row.printed = json.dumps(payload, separators=(",", ":")) row.unit = payload.get("unit") or "" seen_pages: List[str] = [] for ref in refs: page = str(ref.get("page")) if page not in seen_pages: seen_pages.append(page) row.pages = _JOIN.join(seen_pages) row.cited_ids = _JOIN.join(r.get("ref_id", "") for r in refs) # Where a cited answer rests on more than one cell, each column carries # all of them rather than the first. row.cell_text = _JOIN.join(r["text"] for r in refs if r.get("text")) row.status = _JOIN.join(r["status"] for r in refs if r.get("status")) notes = [r["note"] for r in refs if r.get("note")] if notes: row.note = _JOIN.join(([row.note] if row.note else []) + notes) row.review = any(r.get("flagged") for r in refs) return row def percent_format(printed: str) -> str: """A percent format with as many decimals as the figure was printed with, so `6.7 %` shows as 6.7% rather than being padded to 6.70%.""" digits = norm_printed(printed).rstrip("%").strip() places = len(digits.split(".")[1]) if "." in digits else 0 return f"0.{'0' * places}%" if places else "0%" def write_xlsx(state: RunState) -> bytes: """The workbook. Text cells are typed as text — nothing reformats the printed figure and a leading `=` is characters. A percentage's numeric cell holds the fraction under a percent format, so it computes as a rate and displays as the document printed it; the writer picks a value and a format and performs no arithmetic beyond that rescale.""" wb = openpyxl.Workbook() ws = wb.active assert ws is not None ws.title = "batch" ws.append(COLUMNS) for r, row in enumerate(build_rows(state), start=2): texts = { 1: row.question, 2: row.printed, 4: row.unit, 5: row.period, 6: row.pages, 7: row.cited_ids, 8: row.cell_text, 9: row.status, 10: row.note, 11: REVIEW if row.review else "", 12: row.error, } for col, value in texts.items(): if value == "": # A blank cell carries nothing to protect; forcing an empty # string into a text cell reads back as an inline-string None. continue cell = ws.cell(row=r, column=col, value=value) cell.data_type = "s" cell.number_format = "@" if row.number is not None: num = ws.cell(row=r, column=3, value=row.number / 100 if row.percent else row.number) if row.percent: num.number_format = percent_format(row.printed) out = BytesIO() wb.save(out) return out.getvalue() def _quote(s: str) -> str: return '"' + s.replace('"', '""') + '"' def _free_text(s: str) -> str: """A free-text cell: prefixed when it begins with a formula character, then quoted. The prefix is the injection guard; the quotes are what keeps the text as typed when the importer treats quoted fields as text.""" if s[:1] in ("=", "+", "-", "@"): s = "'" + s return _quote(s) def write_csv(state: RunState) -> str: """The same rows as text. The printed column is quoted and never prefixed; the numeric column is bare, and a percentage keeps its percent sign so the importer stores the fraction under a percent format — the same cell the workbook writes directly.""" lines = [",".join(COLUMNS)] for row in build_rows(state): if row.number is None: number = "" elif row.percent: number = f"{row.number}%" else: number = str(row.number) lines.append( ",".join( [ _free_text(row.question), _quote(row.printed) if row.figure else _free_text(row.printed), number, _free_text(row.unit), _free_text(row.period), _quote(row.pages), _quote(row.cited_ids), _free_text(row.cell_text), _quote(row.status), _free_text(row.note), REVIEW if row.review else "", _free_text(row.error), ] ) ) return "\r\n".join(lines) + "\r\n"